diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py new file mode 100644 index 00000000000..b9df6568123 --- /dev/null +++ b/Lib/test/test_asdl_parser.py @@ -0,0 +1,131 @@ +"""Tests for the asdl parser in Parser/asdl.py""" + +import importlib.machinery +import importlib.util +import os +from os.path import dirname +import sys +import sysconfig +import unittest + + +# This test is only relevant for from-source builds of Python. +if not sysconfig.is_python_build(): + raise unittest.SkipTest('test irrelevant for an installed Python') + +src_base = dirname(dirname(dirname(__file__))) +parser_dir = os.path.join(src_base, 'Parser') + + +class TestAsdlParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Loads the asdl module dynamically, since it's not in a real importable + # package. + # Parses Python.asdl into an ast.Module and run the check on it. + # There's no need to do this for each test method, hence setUpClass. + sys.path.insert(0, parser_dir) + loader = importlib.machinery.SourceFileLoader( + 'asdl', os.path.join(parser_dir, 'asdl.py')) + spec = importlib.util.spec_from_loader('asdl', loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + cls.asdl = module + cls.mod = cls.asdl.parse(os.path.join(parser_dir, 'Python.asdl')) + cls.assertTrue(cls.asdl.check(cls.mod), 'Module validation failed') + + @classmethod + def tearDownClass(cls): + del sys.path[0] + + def setUp(self): + # alias stuff from the class, for convenience + self.asdl = TestAsdlParser.asdl + self.mod = TestAsdlParser.mod + self.types = self.mod.types + + def test_module(self): + self.assertEqual(self.mod.name, 'Python') + self.assertIn('stmt', self.types) + self.assertIn('expr', self.types) + self.assertIn('mod', self.types) + + def test_definitions(self): + defs = self.mod.dfns + self.assertIsInstance(defs[0], self.asdl.Type) + self.assertIsInstance(defs[0].value, self.asdl.Sum) + + self.assertIsInstance(self.types['withitem'], self.asdl.Product) + self.assertIsInstance(self.types['alias'], self.asdl.Product) + + def test_product(self): + alias = self.types['alias'] + self.assertEqual( + str(alias), + 'Product([Field(identifier, name), Field(identifier, asname, quantifiers=[OPTIONAL])], ' + '[Field(int, lineno), Field(int, col_offset), ' + 'Field(int, end_lineno, quantifiers=[OPTIONAL]), Field(int, end_col_offset, quantifiers=[OPTIONAL])])') + + def test_attributes(self): + stmt = self.types['stmt'] + self.assertEqual(len(stmt.attributes), 4) + self.assertEqual(repr(stmt.attributes[0]), 'Field(int, lineno)') + self.assertEqual(repr(stmt.attributes[1]), 'Field(int, col_offset)') + self.assertEqual(repr(stmt.attributes[2]), 'Field(int, end_lineno, quantifiers=[OPTIONAL])') + self.assertEqual(repr(stmt.attributes[3]), 'Field(int, end_col_offset, quantifiers=[OPTIONAL])') + + def test_constructor_fields(self): + ehandler = self.types['excepthandler'] + self.assertEqual(len(ehandler.types), 1) + self.assertEqual(len(ehandler.attributes), 4) + + cons = ehandler.types[0] + self.assertIsInstance(cons, self.asdl.Constructor) + self.assertEqual(len(cons.fields), 3) + + f0 = cons.fields[0] + self.assertEqual(f0.type, 'expr') + self.assertEqual(f0.name, 'type') + self.assertTrue(f0.opt) + + f1 = cons.fields[1] + self.assertEqual(f1.type, 'identifier') + self.assertEqual(f1.name, 'name') + self.assertTrue(f1.opt) + + f2 = cons.fields[2] + self.assertEqual(f2.type, 'stmt') + self.assertEqual(f2.name, 'body') + self.assertFalse(f2.opt) + self.assertTrue(f2.seq) + + def test_visitor(self): + class CustomVisitor(self.asdl.VisitorBase): + def __init__(self): + super().__init__() + self.names_with_seq = [] + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type): + self.visit(type.value) + + def visitSum(self, sum): + for t in sum.types: + self.visit(t) + + def visitConstructor(self, cons): + for f in cons.fields: + if f.seq: + self.names_with_seq.append(cons.name) + + v = CustomVisitor() + v.visit(self.types['mod']) + self.assertEqual(v.names_with_seq, + ['Module', 'Module', 'Interactive', 'FunctionType']) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py new file mode 100644 index 00000000000..73bb942af7c --- /dev/null +++ b/Lib/test/test_clinic.py @@ -0,0 +1,4594 @@ +# Argument Clinic +# Copyright 2012-2013 by Larry Hastings. +# Licensed to the PSF under a contributor agreement. + +from functools import partial +from test import support, test_tools +from test.support import os_helper +from test.support.os_helper import TESTFN, unlink, rmtree +from textwrap import dedent +from unittest import TestCase +import inspect +import os.path +import re +import sys +import unittest + +test_tools.skip_if_missing('clinic') +with test_tools.imports_under_tool('clinic'): + import libclinic + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter + from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) + import clinic + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic + + +def repeat_fn(*functions): + def wrapper(test): + def wrapped(self): + for fn in functions: + with self.subTest(fn=fn): + test(self, fn) + return wrapped + return wrapper + +def _make_clinic(*, filename='clinic_tests', limited_capi=False): + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) + return c + + +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, + strip=True): + """Helper for the parser tests. + + tc: unittest.TestCase; passed self in the wrapper + parser: the clinic parser used for this test case + code: a str with input text (clinic code) + errmsg: the expected error message + filename: str, optional filename + lineno: int, optional line number + """ + code = dedent(code) + if strip: + code = code.strip() + errmsg = re.escape(errmsg) + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: + parser(code) + if filename is not None: + tc.assertEqual(cm.exception.filename, filename) + if lineno is not None: + tc.assertEqual(cm.exception.lineno, lineno) + return cm.exception + + +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) + + +class ClinicWholeFileTest(TestCase): + maxDiff = None + + def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): + _expect_failure(self, self.clinic.parse, raw, errmsg, + filename=filename, lineno=lineno) + + def setUp(self): + save_restore_converters(self) + self.clinic = _make_clinic(filename="test.c") + + def test_eol(self): + # regression test: + # clinic's block parser didn't recognize + # the "end line" for the block if it + # didn't end in "\n" (as in, the last) + # byte of the file was '/'. + # so it would spit out an end line for you. + # and since you really already had one, + # the last line of the block got corrupted. + raw = "/*[clinic]\nfoo\n[clinic]*/" + cooked = self.clinic.parse(raw).splitlines() + end_line = cooked[2].rstrip() + # this test is redundant, it's just here explicitly to catch + # the regression test so we don't forget what it looked like + self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") + self.assertEqual(end_line, "[clinic]*/") + + def test_mangled_marker_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: foo]*/ + """ + err = ( + "Mangled Argument Clinic marker line: " + "'/*[clinic end generated code: foo]*/'" + ) + self.expect_failure(raw, err, filename="test.c", lineno=3) + + def test_checksum_mismatch(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ + """ + err = ("Checksum mismatch! " + "Expected '0123456789abcdef', computed 'da39a3ee5e6b4b0d'") + self.expect_failure(raw, err, filename="test.c", lineno=3) + + def test_garbage_after_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/foobarfoobar! + """ + err = "Garbage after stop line: 'foobarfoobar!'" + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_whitespace_before_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + """ + err = ( + "Whitespace is not allowed before the stop line: " + "' [clinic start generated code]*/'" + ) + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_parse_with_body_prefix(self): + clang = CLanguage(None) + clang.body_prefix = "//" + clang.start_line = "//[{dsl_name} start]" + clang.stop_line = "//[{dsl_name} stop]" + cl = Clinic(clang, filename="test.c", limited_capi=False) + raw = dedent(""" + //[clinic start] + //module test + //[clinic stop] + """).strip() + out = cl.parse(raw) + expected = dedent(""" + //[clinic start] + //module test + // + //[clinic stop] + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=65fab8adff58cf08]*/ + """).lstrip() # Note, lstrip() because of the newline + self.assertEqual(out, expected) + + def test_cpp_monitor_fail_nested_block_comment(self): + raw = """ + /* start + /* nested + */ + */ + """ + err = 'Nested block comment!' + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_cpp_monitor_fail_invalid_format_noarg(self): + raw = """ + #if + a() + #endif + """ + err = 'Invalid format for #if line: no argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_cpp_monitor_fail_invalid_format_toomanyargs(self): + raw = """ + #ifdef A B + a() + #endif + """ + err = 'Invalid format for #ifdef line: should be exactly one argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_cpp_monitor_fail_no_matching_if(self): + raw = '#else' + err = '#else without matching #if / #ifdef / #ifndef!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_directive_output_unknown_preset(self): + raw = """ + /*[clinic input] + output preset nosuchpreset + [clinic start generated code]*/ + """ + err = "Unknown preset 'nosuchpreset'" + self.expect_failure(raw, err) + + def test_directive_output_cant_pop(self): + raw = """ + /*[clinic input] + output pop + [clinic start generated code]*/ + """ + err = "Can't 'output pop', stack is empty" + self.expect_failure(raw, err) + + def test_directive_output_print(self): + raw = dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """) + out = self.clinic.parse(raw) + # The generated output will differ for every run, but we can check that + # it starts with the clinic block, we check that it contains all the + # expected fields, and we check that it contains the checksum line. + self.assertStartsWith(out, dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """)) + fields = { + "cpp_endif", + "cpp_if", + "docstring_definition", + "docstring_prototype", + "impl_definition", + "impl_prototype", + "methoddef_define", + "methoddef_ifndef", + "parser_definition", + "parser_prototype", + } + for field in fields: + with self.subTest(field=field): + self.assertIn(field, out) + last_line = out.rstrip().split("\n")[-1] + self.assertStartsWith(last_line, "/*[clinic end generated code: output=") + + def test_directive_wrong_arg_number(self): + raw = dedent(""" + /*[clinic input] + preserve foo bar baz eggs spam ham mushrooms + [clinic start generated code]*/ + """) + err = "takes 1 positional argument but 8 were given" + self.expect_failure(raw, err) + + def test_unknown_destination_command(self): + raw = """ + /*[clinic input] + destination buffer nosuchcommand + [clinic start generated code]*/ + """ + err = "unknown destination command 'nosuchcommand'" + self.expect_failure(raw, err) + + def test_no_access_to_members_in_converter_init(self): + raw = """ + /*[python input] + class Custom_converter(CConverter): + converter = "some_c_function" + def converter_init(self): + self.function.noaccess + [python start generated code]*/ + /*[clinic input] + module test + test.fn + a: Custom + [clinic start generated code]*/ + """ + err = ( + "accessing self.function inside converter_init is disallowed!" + ) + self.expect_failure(raw, err) + + def test_clone_mismatch(self): + err = "'kind' of function and cloned function don't match!" + block = """ + /*[clinic input] + module m + @classmethod + m.f1 + a: object + [clinic start generated code]*/ + /*[clinic input] + @staticmethod + m.f2 = m.f1 + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=9) + + def test_badly_formed_return_annotation(self): + err = "Badly formed annotation for 'm.f': 'Custom'" + block = """ + /*[python input] + class Custom_return_converter(CReturnConverter): + def __init__(self): + raise ValueError("abc") + [python start generated code]*/ + /*[clinic input] + module m + m.f -> Custom + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=8) + + def test_star_after_vararg(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + *args: tuple + * + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_vararg_after_star(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + * + *args: tuple + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_module_already_got_one(self): + err = "Already defined module 'm'!" + block = """ + /*[clinic input] + module m + module m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_already_got_one(self): + err = "Destination already exists: 'test'" + block = """ + /*[clinic input] + destination test new buffer + destination test new buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_does_not_exist(self): + err = "Destination does not exist: '/dev/null'" + block = """ + /*[clinic input] + output everything /dev/null + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_class_already_got_one(self): + err = "Already defined class 'C'!" + block = """ + /*[clinic input] + class C "" "" + class C "" "" + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_cant_nest_module_inside_class(self): + err = "Can't nest a module inside a class!" + block = """ + /*[clinic input] + class C "" "" + module C.m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_dest_buffer_not_empty_at_eof(self): + expected_warning = ("Destination buffer 'buffer' not empty at " + "end of file, emptying.") + expected_generated = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c4668687f5fd002]*/ + + /*[clinic input] + dump buffer + [clinic start generated code]*/ + + PyDoc_VAR(fn__doc__); + + PyDoc_STRVAR(fn__doc__, + "fn($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FN_METHODDEF \\ + {"fn", (PyCFunction)fn, METH_O, fn__doc__}, + + static PyObject * + fn(PyObject *module, PyObject *a) + /*[clinic end generated code: output=be6798b148ab4e53 input=524ce2e021e4eba6]*/ + """) + block = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + """) + with support.captured_stdout() as stdout: + generated = self.clinic.parse(block) + self.assertIn(expected_warning, stdout.getvalue()) + self.assertEqual(generated, expected_generated) + + def test_dest_clear(self): + err = "Can't clear destination 'file': it's not of type 'buffer'" + block = """ + /*[clinic input] + destination file clear + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_misuse(self): + err = "unknown variable 'ets'" + block = """ + /*[clinic input] + set ets tse + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_prefix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '// ' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "// PyDoc_VAR(fn__doc__);" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_suffix(self): + block = dedent(""" + /*[clinic input] + set line_suffix ' // test' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "PyDoc_VAR(fn__doc__); // test" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_prefix_and_suffix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '{block comment start} ' + set line_suffix ' {block comment end}' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "/* PyDoc_VAR(fn__doc__); */" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_printout(self): + block = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + """) + expected = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + test + /*[clinic end generated code: output=4e1243bd22c66e76 input=898f1a32965d44ca]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, expected) + + def test_directive_preserve_twice(self): + err = "Can't have 'preserve' twice in one block!" + block = """ + /*[clinic input] + preserve + preserve + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_directive_preserve_input(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = """ + /*[clinic input] + preserve + fn + a: object + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_directive_preserve_output(self): + block = dedent(""" + /*[clinic input] + output everything buffer + preserve + [clinic start generated code]*/ + // Preserve this + /*[clinic end generated code: output=eaa49677ae4c1f7d input=559b5db18fddae6a]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, block) + + def test_directive_output_invalid_command(self): + err = dedent(""" + Invalid command or destination name 'cmd'. Must be one of: + - 'preset' + - 'push' + - 'pop' + - 'print' + - 'everything' + - 'cpp_if' + - 'docstring_prototype' + - 'docstring_definition' + - 'methoddef_define' + - 'impl_prototype' + - 'parser_prototype' + - 'parser_definition' + - 'cpp_endif' + - 'methoddef_ifndef' + - 'impl_definition' + """).strip() + block = """ + /*[clinic input] + output cmd buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_validate_cloned_init(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + @classmethod + C.__init__ = C.meth + [clinic start generated code]*/ + """ + err = "'__init__' must be a normal method; got 'FunctionKind.CLASS_METHOD'!" + self.expect_failure(block, err, lineno=8) + + def test_validate_cloned_new(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + C.__new__ = C.meth + [clinic start generated code]*/ + """ + err = "'__new__' must be a class method" + self.expect_failure(block, err, lineno=7) + + def test_no_c_basename_cloned(self): + block = """ + /*[clinic input] + foo2 + [clinic start generated code]*/ + /*[clinic input] + foo as = foo2 + [clinic start generated code]*/ + """ + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, lineno=5) + + def test_cloned_with_custom_c_basename(self): + raw = dedent(""" + /*[clinic input] + # Make sure we don't create spurious clinic/ directories. + output everything suppress + foo2 + [clinic start generated code]*/ + + /*[clinic input] + foo as foo1 = foo2 + [clinic start generated code]*/ + """) + self.clinic.parse(raw) + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + self.assertEqual(funcs[1].name, "foo") + self.assertEqual(funcs[1].c_basename, "foo1") + + def test_cloned_with_illegal_c_basename(self): + block = """ + /*[clinic input] + class C "void *" "" + foo1 + [clinic start generated code]*/ + + /*[clinic input] + foo2 as .illegal. = foo1 + [clinic start generated code]*/ + """ + err = "Illegal C basename: '.illegal.'" + self.expect_failure(block, err, lineno=7) + + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + + def test_ignore_preprocessor_in_comments(self): + for dsl in "clinic", "python": + raw = dedent(f"""\ + /*[{dsl} input] + # CPP directives, valid or not, should be ignored in C comments. + # + [{dsl} start generated code]*/ + """) + self.clinic.parse(raw) + + +class ParseFileUnitTest(TestCase): + def expect_parsing_failure( + self, *, filename, expected_error, verify=True, output=None + ): + errmsg = re.escape(dedent(expected_error).strip()) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) + + def test_parse_file_no_extension(self) -> None: + self.expect_parsing_failure( + filename="foo", + expected_error="Can't extract file type for file 'foo'" + ) + + def test_parse_file_strange_extension(self) -> None: + filenames_to_errors = { + "foo.rs": "Can't identify file type for file 'foo.rs'", + "foo.hs": "Can't identify file type for file 'foo.hs'", + "foo.js": "Can't identify file type for file 'foo.js'", + } + for filename, errmsg in filenames_to_errors.items(): + with self.subTest(filename=filename): + self.expect_parsing_failure(filename=filename, expected_error=errmsg) + + +class ClinicGroupPermuterTest(TestCase): + def _test(self, l, m, r, output): + computed = permute_optional_groups(l, m, r) + self.assertEqual(output, computed) + + def test_range(self): + self._test([['start']], ['stop'], [['step']], + ( + ('stop',), + ('start', 'stop',), + ('start', 'stop', 'step',), + )) + + def test_add_window(self): + self._test([['x', 'y']], ['ch'], [['attr']], + ( + ('ch',), + ('ch', 'attr'), + ('x', 'y', 'ch',), + ('x', 'y', 'ch', 'attr'), + )) + + def test_ludicrous(self): + self._test([['a1', 'a2', 'a3'], ['b1', 'b2']], ['c1'], [['d1', 'd2'], ['e1', 'e2', 'e3']], + ( + ('c1',), + ('b1', 'b2', 'c1'), + ('b1', 'b2', 'c1', 'd1', 'd2'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1', 'd1', 'd2'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1', 'd1', 'd2', 'e1', 'e2', 'e3'), + )) + + def test_right_only(self): + self._test([], [], [['a'],['b'],['c']], + ( + (), + ('a',), + ('a', 'b'), + ('a', 'b', 'c') + )) + + def test_have_left_options_but_required_is_empty(self): + def fn(): + permute_optional_groups(['a'], [], []) + self.assertRaises(ValueError, fn) + + +class ClinicLinearFormatTest(TestCase): + def _test(self, input, output, **kwargs): + computed = libclinic.linear_format(input, **kwargs) + self.assertEqual(output, computed) + + def test_empty_strings(self): + self._test('', '') + + def test_solo_newline(self): + self._test('\n', '\n') + + def test_no_substitution(self): + self._test(""" + abc + """, """ + abc + """) + + def test_empty_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + def + """, name='') + + def test_single_line_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + GARGLE + def + """, name='GARGLE') + + def test_multiline_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + bingle + bungle + + def + """, name='bingle\nbungle\n') + + def test_text_before_block_marker(self): + regex = re.escape("found before '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("no text before marker for you! {marker}", + marker="not allowed!") + + def test_text_after_block_marker(self): + regex = re.escape("found after '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("{marker} no text after marker for you!", + marker="not allowed!") + + +class InertParser: + def __init__(self, clinic): + pass + + def parse(self, block): + pass + +class CopyParser: + def __init__(self, clinic): + pass + + def parse(self, block): + block.output = block.input + + +class ClinicBlockParserTest(TestCase): + def _test(self, input, output): + language = CLanguage(None) + + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) + for block in blocks: + writer.print_block(block) + output = writer.f.getvalue() + assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) + + def round_trip(self, input): + return self._test(input, input) + + def test_round_trip_1(self): + self.round_trip(""" + verbatim text here + lah dee dah + """) + def test_round_trip_2(self): + self.round_trip(""" + verbatim text here + lah dee dah +/*[inert] +abc +[inert]*/ +def +/*[inert checksum: 7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ +xyz +""") + + def _test_clinic(self, input, output): + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) + c.parsers['inert'] = InertParser(c) + c.parsers['copy'] = CopyParser(c) + computed = c.parse(input) + self.assertEqual(output, computed) + + def test_clinic_1(self): + self._test_clinic(""" + verbatim text here + lah dee dah +/*[copy input] +def +[copy start generated code]*/ +abc +/*[copy end generated code: output=03cfd743661f0797 input=7b18d017f89f61cf]*/ +xyz +""", """ + verbatim text here + lah dee dah +/*[copy input] +def +[copy start generated code]*/ +def +/*[copy end generated code: output=7b18d017f89f61cf input=7b18d017f89f61cf]*/ +xyz +""") + + +class ClinicParserTest(TestCase): + + def parse(self, text): + c = _make_clinic() + parser = DSLParser(c) + block = Block(text) + parser.parse(block) + return block + + def parse_function(self, text, signatures_in_block=2, function_index=1): + block = self.parse(text) + s = block.signatures + self.assertEqual(len(s), signatures_in_block) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) + return s[function_index] + + def expect_failure(self, block, err, *, + filename=None, lineno=None, strip=True): + return _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno, strip=strip) + + def checkDocstring(self, fn, expected): + self.assertTrue(hasattr(fn, "docstring")) + self.assertEqual(dedent(expected).strip(), + fn.docstring.strip()) + + def test_trivial(self): + parser = DSLParser(_make_clinic()) + block = Block(""" + module os + os.access + """) + parser.parse(block) + module, function = block.signatures + self.assertEqual("access", function.name) + self.assertEqual("os", module.name) + + def test_ignore_line(self): + block = self.parse(dedent(""" + # + module os + os.access + """)) + module, function = block.signatures + self.assertEqual("access", function.name) + self.assertEqual("os", module.name) + + def test_param(self): + function = self.parse_function(""" + module os + os.access + path: int + """) + self.assertEqual("access", function.name) + self.assertEqual(2, len(function.parameters)) + p = function.parameters['path'] + self.assertEqual('path', p.name) + self.assertIsInstance(p.converter, int_converter) + + def test_param_default(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: bool = True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(True, p.default) + + def test_param_with_continuations(self): + function = self.parse_function(r""" + module os + os.access + follow_symlinks: \ + bool \ + = \ + True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(True, p.default) + + def test_param_default_none(self): + function = self.parse_function(r""" + module test + test.func + obj: object = None + str: str(accept={str, NoneType}) = None + buf: Py_buffer(accept={str, buffer, NoneType}) = None + """) + p = function.parameters['obj'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'Py_None') + + p = function.parameters['str'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + def test_param_default_null(self): + function = self.parse_function(r""" + module test + test.func + obj: object = NULL + str: str = NULL + buf: Py_buffer = NULL + fsencoded: unicode_fs_encoded = NULL + fsdecoded: unicode_fs_decoded = NULL + """) + p = function.parameters['obj'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['str'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + p = function.parameters['fsencoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['fsdecoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + def test_param_default_str_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str = ' \t\n\r\v\f\xa0' + buf: Py_buffer(accept={str, buffer}) = ' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\u00a0"') + + p = function.parameters['buf'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\302\240", .obj = NULL, .len = 8}') + + def test_param_default_bytes_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str(accept={robuffer}) = b' \t\n\r\v\f\xa0' + buf: Py_buffer = b' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\240"') + + p = function.parameters['buf'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\240", .obj = NULL, .len = 7}') + + def test_param_default_byte_literal(self): + function = self.parse_function(r""" + module test + test.func + zero: char = b'\0' + one: char = b'\1' + lf: char = b'\n' + nbsp: char = b'\xa0' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, b'\0') + self.assertEqual(p.converter.py_default, r"b'\x00'") + self.assertEqual(p.converter.c_default, r"'\0'") + + p = function.parameters['one'] + self.assertEqual(p.default, b'\1') + self.assertEqual(p.converter.py_default, r"b'\x01'") + self.assertEqual(p.converter.c_default, r"'\001'") + + p = function.parameters['lf'] + self.assertEqual(p.default, b'\n') + self.assertEqual(p.converter.py_default, r"b'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, b'\xa0') + self.assertEqual(p.converter.py_default, r"b'\xa0'") + self.assertEqual(p.converter.c_default, r"'\240'") + + def test_param_default_unicode_char(self): + function = self.parse_function(r""" + module test + test.func + zero: int(accept={str}) = '\0' + one: int(accept={str}) = '\1' + lf: int(accept={str}) = '\n' + nbsp: int(accept={str}) = '\xa0' + snake: int(accept={str}) = '\U0001f40d' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, '\0') + self.assertEqual(p.converter.py_default, r"'\x00'") + self.assertEqual(p.converter.c_default, '0') + + p = function.parameters['one'] + self.assertEqual(p.default, '\1') + self.assertEqual(p.converter.py_default, r"'\x01'") + self.assertEqual(p.converter.c_default, '0x01') + + p = function.parameters['lf'] + self.assertEqual(p.default, '\n') + self.assertEqual(p.converter.py_default, r"'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, '\xa0') + self.assertEqual(p.converter.py_default, r"'\xa0'") + self.assertEqual(p.converter.c_default, '0xa0') + + p = function.parameters['snake'] + self.assertEqual(p.default, '\U0001f40d') + self.assertEqual(p.converter.py_default, "'\U0001f40d'") + self.assertEqual(p.converter.c_default, '0x1f40d') + + def test_param_default_bool(self): + function = self.parse_function(r""" + module test + test.func + bool: bool = True + intbool: bool(accept={int}) = True + intbool2: bool(accept={int}) = 2 + """) + p = function.parameters['bool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool2'] + self.assertEqual(p.default, 2) + self.assertEqual(p.converter.py_default, '2') + self.assertEqual(p.converter.c_default, '2') + + def test_param_default_expr_named_constant(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = sys.maxsize + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(sys.maxsize, p.default) + self.assertEqual("MAXSIZE", p.converter.c_default) + + err = ( + "When you specify a named constant ('sys.maxsize') as your default value, " + "you MUST specify a valid c_default." + ) + block = """ + module os + os.access + follow_symlinks: int = sys.maxsize + """ + self.expect_failure(block, err, lineno=2) + + def test_param_with_bizarre_default_fails_correctly(self): + template = """ + module os + os.access + follow_symlinks: int = {default} + """ + err = "Unsupported expression as default value" + for bad_default_value in ( + "{1, 2, 3}", + "3 if bool() else 4", + "[x for x in range(42)]" + ): + with self.subTest(bad_default=bad_default_value): + block = template.format(default=bad_default_value) + self.expect_failure(block, err, lineno=2) + + def test_unspecified_not_allowed_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = unspecified + """ + err = "'unspecified' is not a legal default value!" + exc = self.expect_failure(block, err, lineno=2) + self.assertNotIn('Malformed expression given as default value', str(exc)) + + def test_malformed_expression_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = 1/0 + """ + err = "Malformed expression given as default value" + self.expect_failure(block, err, lineno=2) + + def test_param_default_expr_binop(self): + err = ( + "When you specify an expression ('a + b') as your default value, " + "you MUST specify a valid c_default." + ) + block = """ + fn + follow_symlinks: int = a + b + """ + self.expect_failure(block, err, lineno=1) + + def test_param_no_docstring(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: bool = True + something_else: str = '' + """) + self.assertEqual(3, len(function.parameters)) + conv = function.parameters['something_else'].converter + self.assertIsInstance(conv, str_converter) + + def test_param_default_parameters_out_of_order(self): + err = ( + "Can't have a parameter without a default ('something_else') " + "after a parameter with a default!" + ) + block = """ + module os + os.access + follow_symlinks: bool = True + something_else: str + """ + self.expect_failure(block, err, lineno=3) + + def disabled_test_converter_arguments(self): + function = self.parse_function(""" + module os + os.access + path: path_t(allow_fd=1) + """) + p = function.parameters['path'] + self.assertEqual(1, p.converter.args['allow_fd']) + + def test_function_docstring(self): + function = self.parse_function(""" + module os + os.stat as os_stat_fn + + path: str + Path to be examined + Ensure that multiple lines are indented correctly. + + Perform a stat system call on the given path. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. + """) + self.checkDocstring(function, """ + stat($module, /, path) + -- + + Perform a stat system call on the given path. + + path + Path to be examined + Ensure that multiple lines are indented correctly. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. + """) + + def test_docstring_trailing_whitespace(self): + function = self.parse_function( + "module t\n" + "t.s\n" + " a: object\n" + " Param docstring with trailing whitespace \n" + "Func docstring summary with trailing whitespace \n" + " \n" + "Func docstring body with trailing whitespace \n" + ) + self.checkDocstring(function, """ + s($module, /, a) + -- + + Func docstring summary with trailing whitespace + + a + Param docstring with trailing whitespace + + Func docstring body with trailing whitespace + """) + + def test_explicit_parameters_in_docstring(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + Documentation for x. + y: int + + This is the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + + def test_docstring_with_comments(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + # We're about to have + # the documentation for x. + Documentation for x. + # We've just had + # the documentation for x. + y: int + + # We're about to have + # the documentation for foo. + This is the documentation for foo. + # We've just had + # the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + + def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): + function = self.parse_function(dedent(""" + module os + os.stat + path: str + This/used to break Clinic! + """)) + self.checkDocstring(function, """ + stat($module, /, path) + -- + + This/used to break Clinic! + """) + + def test_c_name(self): + function = self.parse_function(""" + module os + os.stat as os_stat_fn + """) + self.assertEqual("os_stat_fn", function.c_basename) + + def test_base_invalid_syntax(self): + block = """ + module os + os.stat + invalid syntax: int = 42 + """ + err = "Function 'stat' has an invalid parameter declaration: 'invalid syntax: int = 42'" + self.expect_failure(block, err, lineno=2) + + def test_param_default_invalid_syntax(self): + block = """ + module os + os.stat + x: int = invalid syntax + """ + err = "Function 'stat' has an invalid parameter declaration:" + self.expect_failure(block, err, lineno=2) + + def test_cloning_nonexistent_function_correctly_fails(self): + block = """ + cloned = fooooooooooooooooo + This is trying to clone a nonexistent function!! + """ + err = "Couldn't find existing function 'fooooooooooooooooo'!" + with support.captured_stderr() as stderr: + self.expect_failure(block, err, lineno=0) + expected_debug_print = dedent("""\ + cls=None, module=, existing='fooooooooooooooooo' + (cls or module).functions=[] + """) + stderr = stderr.getvalue() + self.assertIn(expected_debug_print, stderr) + + def test_return_converter(self): + function = self.parse_function(""" + module os + os.stat -> int + """) + self.assertIsInstance(function.return_converter, int_return_converter) + + def test_return_converter_invalid_syntax(self): + block = """ + module os + os.stat -> invalid syntax + """ + err = "Badly formed annotation for 'os.stat': 'invalid syntax'" + self.expect_failure(block, err) + + def test_legacy_converter_disallowed_in_return_annotation(self): + block = """ + module os + os.stat -> "s" + """ + err = "Legacy converter 's' not allowed as a return converter" + self.expect_failure(block, err) + + def test_unknown_return_converter(self): + block = """ + module os + os.stat -> fooooooooooooooooooooooo + """ + err = "No available return converter called 'fooooooooooooooooooooooo'" + self.expect_failure(block, err) + + def test_star(self): + function = self.parse_function(""" + module os + os.access + * + follow_symlinks: bool = True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(inspect.Parameter.KEYWORD_ONLY, p.kind) + self.assertEqual(0, p.group) + + def test_group(self): + function = self.parse_function(""" + module window + window.border + [ + ls: int + ] + / + """) + p = function.parameters['ls'] + self.assertEqual(1, p.group) + + def test_left_group(self): + function = self.parse_function(""" + module curses + curses.addch + [ + y: int + Y-coordinate. + x: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr: long + Attributes for the character. + ] + / + """) + dataset = ( + ('y', -1), ('x', -1), + ('ch', 0), + ('attr', 1), + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + self.checkDocstring(function, """ + addch([y, x,] ch, [attr]) + + + y + Y-coordinate. + x + X-coordinate. + ch + Character to add. + attr + Attributes for the character. + """) + + def test_nested_groups(self): + function = self.parse_function(""" + module curses + curses.imaginary + [ + [ + y1: int + Y-coordinate. + y2: int + Y-coordinate. + ] + x1: int + X-coordinate. + x2: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr1: long + Attributes for the character. + attr2: long + Attributes for the character. + attr3: long + Attributes for the character. + [ + attr4: long + Attributes for the character. + attr5: long + Attributes for the character. + attr6: long + Attributes for the character. + ] + ] + / + """) + dataset = ( + ('y1', -2), ('y2', -2), + ('x1', -1), ('x2', -1), + ('ch', 0), + ('attr1', 1), ('attr2', 1), ('attr3', 1), + ('attr4', 2), ('attr5', 2), ('attr6', 2), + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + self.checkDocstring(function, """ + imaginary([[y1, y2,] x1, x2,] ch, [attr1, attr2, attr3, [attr4, attr5, + attr6]]) + + + y1 + Y-coordinate. + y2 + Y-coordinate. + x1 + X-coordinate. + x2 + X-coordinate. + ch + Character to add. + attr1 + Attributes for the character. + attr2 + Attributes for the character. + attr3 + Attributes for the character. + attr4 + Attributes for the character. + attr5 + Attributes for the character. + attr6 + Attributes for the character. + """) + + def test_disallowed_grouping__two_top_groups_on_left(self): + err = ( + "Function 'two_top_groups_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" + ) + block = """ + module foo + foo.two_top_groups_on_left + [ + group1 : int + ] + [ + group2 : int + ] + param: int + """ + self.expect_failure(block, err, lineno=5) + + def test_disallowed_grouping__two_top_groups_on_right(self): + block = """ + module foo + foo.two_top_groups_on_right + param: int + [ + group1 : int + ] + [ + group2 : int + ] + """ + err = ( + "Function 'two_top_groups_on_right' has an unsupported group " + "configuration. (Unexpected state 6.b)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__parameter_after_group_on_right(self): + block = """ + module foo + foo.parameter_after_group_on_right + param: int + [ + [ + group1 : int + ] + group2 : int + ] + """ + err = ( + "Function parameter_after_group_on_right has an unsupported group " + "configuration. (Unexpected state 6.a)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__group_after_parameter_on_left(self): + block = """ + module foo + foo.group_after_parameter_on_left + [ + group2 : int + [ + group1 : int + ] + ] + param: int + """ + err = ( + "Function 'group_after_parameter_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__empty_group_on_left(self): + block = """ + module foo + foo.empty_group + [ + [ + ] + group2 : int + ] + param: int + """ + err = ( + "Function 'empty_group' has an empty group. " + "All groups must contain at least one parameter." + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__empty_group_on_right(self): + block = """ + module foo + foo.empty_group + param: int + [ + [ + ] + group2 : int + ] + """ + err = ( + "Function 'empty_group' has an empty group. " + "All groups must contain at least one parameter." + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__no_matching_bracket(self): + block = """ + module foo + foo.empty_group + param: int + ] + group2: int + ] + """ + err = "Function 'empty_group' has a ']' without a matching '['" + self.expect_failure(block, err) + + def test_disallowed_grouping__must_be_position_only(self): + dataset = (""" + with_kwds + [ + * + a: object + ] + """, """ + with_kwds + [ + a: object + ] + """) + err = ( + "You cannot use optional groups ('[' and ']') unless all " + "parameters are positional-only ('/')" + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_no_parameters(self): + function = self.parse_function(""" + module foo + foo.bar + + Docstring + + """) + self.assertEqual("bar($module, /)\n--\n\nDocstring", function.docstring) + self.assertEqual(1, len(function.parameters)) # self! + + def test_init_with_no_parameters(self): + function = self.parse_function(""" + module foo + class foo.Bar "unused" "notneeded" + foo.Bar.__init__ + + Docstring + + """, signatures_in_block=3, function_index=2) + + # self is not in the signature + self.assertEqual("Bar()\n--\n\nDocstring", function.docstring) + # but it *is* a parameter + self.assertEqual(1, len(function.parameters)) + + def test_illegal_module_line(self): + block = """ + module foo + foo.bar => int + / + """ + err = "Illegal function name: 'foo.bar => int'" + self.expect_failure(block, err) + + def test_illegal_c_basename(self): + block = """ + module foo + foo.bar as 935 + / + """ + err = "Illegal C basename: '935'" + self.expect_failure(block, err) + + def test_no_c_basename(self): + block = "foo as " + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, strip=False) + + def test_single_star(self): + block = """ + module foo + foo.bar + * + * + """ + err = "Function 'bar' uses '*' more than once." + self.expect_failure(block, err) + + def test_parameters_required_after_star(self): + dataset = ( + "module foo\nfoo.bar\n *", + "module foo\nfoo.bar\n *\nDocstring here.", + "module foo\nfoo.bar\n this: int\n *", + "module foo\nfoo.bar\n this: int\n *\nDocstring.", + ) + err = "Function 'bar' specifies '*' without following parameters." + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_fulldisplayname_class(self): + dataset = ( + ("T", """ + class T "void *" "" + T.__init__ + """), + ("m.T", """ + module m + class m.T "void *" "" + @classmethod + m.T.__new__ + """), + ("m.T.C", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.__init__ + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_fulldisplayname_meth(self): + dataset = ( + ("func", "func"), + ("m.func", """ + module m + m.func + """), + ("T.meth", """ + class T "void *" "" + T.meth + """), + ("m.T.meth", """ + module m + class m.T "void *" "" + m.T.meth + """), + ("m.T.C.meth", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.meth + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_depr_star_invalid_format_1(self): + block = """ + module foo + foo.bar + this: int + * [from 3] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got '3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_2(self): + block = """ + module foo + foo.bar + this: int + * [from a.b] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got 'a.b'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_3(self): + block = """ + module foo + foo.bar + this: int + * [from 1.2.3] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got '1.2.3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_parameters_required_after_depr_star(self): + block = """ + module foo + foo.bar + this: int + * [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + * + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star3(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + *args: tuple + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_star(self): + block = """ + module foo + foo.bar + a: int + * + * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_vararg(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_duplicate(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '* [from 3.14]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_star_duplicate2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar': '* [from 3.15]' must precede '* [from 3.14]'" + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from 3.14]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate2(self): + block = """ + module foo + foo.bar + a: int + / [from 3.15] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" + self.expect_failure(block, err, lineno=5) + + def test_single_slash(self): + block = """ + module foo + foo.bar + / + / + """ + err = ( + "Function 'bar' has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.expect_failure(block, err) + + def test_parameters_required_before_depr_slash(self): + block = """ + module foo + foo.bar + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=2) + + def test_parameters_required_before_depr_slash2(self): + block = """ + module foo + foo.bar + a: int + / + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_double_slash(self): + block = """ + module foo + foo.bar + a: int + / + b: int + / + """ + err = "Function 'bar' uses '/' more than once." + self.expect_failure(block, err) + + def test_slash_after_star(self): + block = """ + module foo + foo.bar + x: int + y: int + * + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + + def test_slash_after_vararg(self): + block = """ + module foo + foo.bar + x: int + y: int + *args: tuple + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + + def test_depr_star_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_vararg_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_slash_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '/ [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_parameters_not_permitted_after_slash_for_now(self): + block = """ + module foo + foo.bar + / + x: int + """ + err = ( + "Function 'bar' has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.expect_failure(block, err) + + def test_parameters_no_more_than_one_vararg(self): + err = "Function 'bar' uses '*' more than once." + block = """ + module foo + foo.bar + *vararg1: tuple + *vararg2: tuple + """ + self.expect_failure(block, err, lineno=3) + + def test_function_not_at_column_0(self): + function = self.parse_function(""" + module foo + foo.bar + x: int + Nested docstring here, goeth. + * + y: str + Not at column 0! + """) + self.checkDocstring(function, """ + bar($module, /, x, *, y) + -- + + Not at column 0! + + x + Nested docstring here, goeth. + """) + + def test_docstring_only_summary(self): + function = self.parse_function(""" + module m + m.f + summary + """) + self.checkDocstring(function, """ + f($module, /) + -- + + summary + """) + + def test_docstring_empty_lines(self): + function = self.parse_function(""" + module m + m.f + + + """) + self.checkDocstring(function, """ + f($module, /) + -- + """) + + def test_docstring_explicit_params_placement(self): + function = self.parse_function(""" + module m + m.f + a: int + Param docstring for 'a' will be included + b: int + c: int + Param docstring for 'c' will be included + This is the summary line. + + We'll now place the params section here: + {parameters} + And now for something completely different! + (Note the added newline) + """) + self.checkDocstring(function, """ + f($module, /, a, b, c) + -- + + This is the summary line. + + We'll now place the params section here: + a + Param docstring for 'a' will be included + c + Param docstring for 'c' will be included + + And now for something completely different! + (Note the added newline) + """) + + def test_indent_stack_no_tabs(self): + block = """ + module foo + foo.bar + *vararg1: tuple + \t*vararg2: tuple + """ + err = ("Tab characters are illegal in the Clinic DSL: " + r"'\t*vararg2: tuple'") + self.expect_failure(block, err) + + def test_indent_stack_illegal_outdent(self): + block = """ + module foo + foo.bar + a: object + b: object + """ + err = "Illegal outdent" + self.expect_failure(block, err) + + def test_directive(self): + parser = DSLParser(_make_clinic()) + parser.flag = False + parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) + block = Block("setflag") + parser.parse(block) + self.assertTrue(parser.flag) + + def test_legacy_converters(self): + block = self.parse('module os\nos.access\n path: "s"') + module, function = block.signatures + conv = (function.parameters['path']).converter + self.assertIsInstance(conv, str_converter) + + def test_legacy_converters_non_string_constant_annotation(self): + err = "Annotations must be either a name, a function call, or a string" + dataset = ( + 'module os\nos.access\n path: 42', + 'module os\nos.access\n path: 42.42', + 'module os\nos.access\n path: 42j', + 'module os\nos.access\n path: b"42"', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err, lineno=2) + + def test_other_bizarre_things_in_annotations_fail(self): + err = "Annotations must be either a name, a function call, or a string" + dataset = ( + 'module os\nos.access\n path: {"some": "dictionary"}', + 'module os\nos.access\n path: ["list", "of", "strings"]', + 'module os\nos.access\n path: (x for x in range(42))', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err, lineno=2) + + def test_kwarg_splats_disallowed_in_function_call_annotations(self): + err = "Cannot use a kwarg splat in a function-call annotation" + dataset = ( + 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{"bang": 42})', + 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_self_param_placement(self): + err = ( + "A 'self' parameter, if specified, must be the very first thing " + "in the parameter block." + ) + block = """ + module foo + foo.func + a: int + self: self(type="PyObject *") + """ + self.expect_failure(block, err, lineno=3) + + def test_self_param_cannot_be_optional(self): + err = "A 'self' parameter cannot be marked optional." + block = """ + module foo + foo.func + self: self(type="PyObject *") = None + """ + self.expect_failure(block, err, lineno=2) + + def test_defining_class_param_placement(self): + err = ( + "A 'defining_class' parameter, if specified, must either be the " + "first thing in the parameter block, or come just after 'self'." + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") + a: int + cls: defining_class + """ + self.expect_failure(block, err, lineno=4) + + def test_defining_class_param_cannot_be_optional(self): + err = "A 'defining_class' parameter cannot be marked optional." + block = """ + module foo + foo.func + cls: defining_class(type="PyObject *") = None + """ + self.expect_failure(block, err, lineno=2) + + def test_slot_methods_cannot_access_defining_class(self): + block = """ + module foo + class Foo "" "" + Foo.__init__ + cls: defining_class + a: object + """ + err = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, err): + self.parse_function(block) + + def test_new_must_be_a_class_method(self): + err = "'__new__' must be a class method!" + block = """ + module foo + class Foo "" "" + Foo.__new__ + """ + self.expect_failure(block, err, lineno=2) + + def test_init_must_be_a_normal_method(self): + err_template = "'__init__' must be a normal method; got 'FunctionKind.{}'!" + annotations = { + "@classmethod": "CLASS_METHOD", + "@staticmethod": "STATIC_METHOD", + "@getter": "GETTER", + } + for annotation, invalid_kind in annotations.items(): + with self.subTest(annotation=annotation, invalid_kind=invalid_kind): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.__init__ + """ + expected_error = err_template.format(invalid_kind) + self.expect_failure(block, expected_error, lineno=3) + + def test_init_cannot_define_a_return_type(self): + block = """ + class Foo "" "" + Foo.__init__ -> long + """ + expected_error = "__init__ methods cannot define a return type" + self.expect_failure(block, expected_error, lineno=1) + + def test_invalid_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property -> int + """ + expected_error = f"{annotation} method cannot define a return type" + self.expect_failure(block, expected_error, lineno=3) + + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property + obj: int + / + """ + expected_error = f"{annotation} methods cannot define parameters" + self.expect_failure(block, expected_error) + + def test_setter_docstring(self): + block = """ + module foo + class Foo "" "" + @setter + Foo.property + + foo + + bar + [clinic start generated code]*/ + """ + expected_error = "docstrings are only supported for @getter, not @setter" + self.expect_failure(block, expected_error) + + def test_duplicate_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + {annotation} + Foo.property -> int + """ + expected_error = f"Cannot apply {annotation} twice to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getter_and_setter_disallowed_on_same_function(self): + dup_annotations = [("@getter", "@setter"), ("@setter", "@getter")] + for dup in dup_annotations: + with self.subTest(dup=dup): + block = f""" + module foo + class Foo "" "" + {dup[0]} + {dup[1]} + Foo.property -> int + """ + expected_error = "Cannot apply both @getter and @setter to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getset_no_class(self): + for annotation in "@getter", "@setter": + with self.subTest(annotation=annotation): + block = f""" + module m + {annotation} + m.func + """ + expected_error = "@getter and @setter must be methods" + self.expect_failure(block, expected_error, lineno=2) + + def test_duplicate_coexist(self): + err = "Called @coexist twice" + block = """ + module m + @coexist + @coexist + m.fn + """ + self.expect_failure(block, err, lineno=2) + + def test_unused_param(self): + block = self.parse(""" + module foo + foo.func + fn: object + k: float + i: float(unused=True) + / + * + flag: bool(unused=True) = False + """) + sig = block.signatures[1] # Function index == 1 + params = sig.parameters + conv = lambda fn: params[fn].converter + dataset = ( + {"name": "fn", "unused": False}, + {"name": "k", "unused": False}, + {"name": "i", "unused": True}, + {"name": "flag", "unused": True}, + ) + for param in dataset: + name, unused = param.values() + with self.subTest(name=name, unused=unused): + p = conv(name) + # Verify that the unused flag is parsed correctly. + self.assertEqual(unused, p.unused) + + # Now, check that we'll produce correct code. + decl = p.simple_declaration(in_parser=False) + if unused: + self.assertIn("Py_UNUSED", decl) + else: + self.assertNotIn("Py_UNUSED", decl) + + # Make sure the Py_UNUSED macro is not used in the parser body. + parser_decl = p.simple_declaration(in_parser=True) + self.assertNotIn("Py_UNUSED", parser_decl) + + def test_scaffolding(self): + # test repr on special values + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') + + # test that fail fails + with support.captured_stdout() as stdout: + errmsg = 'The igloos are melting' + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) + exc = cm.exception + self.assertEqual(exc.filename, 'clown.txt') + self.assertEqual(exc.lineno, 69) + self.assertEqual(stdout.getvalue(), "") + + def test_non_ascii_character_in_docstring(self): + block = """ + module test + test.fn + a: int + á param docstring + docstring fü bár baß + """ + with support.captured_stdout() as stdout: + self.parse(block) + # The line numbers are off; this is a known limitation. + expected = dedent("""\ + Warning: + Non-ascii characters are not allowed in docstrings: 'á' + + Warning: + Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' + + """) + self.assertEqual(stdout.getvalue(), expected) + + def test_illegal_c_identifier(self): + err = "Illegal C identifier: 17a" + block = """ + module test + test.fn + a as 17a: int + """ + self.expect_failure(block, err, lineno=2) + + def test_cannot_convert_special_method(self): + err = "'__len__' is a special method and cannot be converted" + block = """ + class T "" "" + T.__len__ + """ + self.expect_failure(block, err, lineno=1) + + def test_cannot_specify_pydefault_without_default(self): + err = "You can't specify py_default without specifying a default value!" + block = """ + fn + a: object(py_default='NULL') + """ + self.expect_failure(block, err, lineno=1) + + def test_vararg_cannot_take_default_value(self): + err = "Function 'fn' has an invalid parameter declaration:" + block = """ + fn + *args: tuple = None + """ + self.expect_failure(block, err, lineno=1) + + def test_default_is_not_of_correct_type(self): + err = ("int_converter: default value 2.5 for field 'a' " + "is not of type 'int'") + block = """ + fn + a: int = 2.5 + """ + self.expect_failure(block, err, lineno=1) + + def test_invalid_legacy_converter(self): + err = "'fhi' is not a valid legacy converter" + block = """ + fn + a: 'fhi' + """ + self.expect_failure(block, err, lineno=1) + + def test_parent_class_or_module_does_not_exist(self): + err = "Parent class or module 'baz' does not exist" + block = """ + module m + baz.func + """ + self.expect_failure(block, err, lineno=1) + + def test_duplicate_param_name(self): + err = "You can't have two parameters named 'a'" + block = """ + module m + m.func + a: int + a: float + """ + self.expect_failure(block, err, lineno=3) + + def test_param_requires_custom_c_name(self): + err = "Parameter 'module' requires a custom C name" + block = """ + module m + m.func + module: int + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_assert_no_group(self): + err = "Function 'func' has a ']' without a matching '['" + block = """ + module m + m.func + ] + docstring + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_no_summary(self): + err = "Docstring for 'm.func' does not have a summary line!" + block = """ + module m + m.func + docstring1 + docstring2 + """ + self.expect_failure(block, err, lineno=3) + + def test_state_func_docstring_only_one_param_template(self): + err = "You may not specify {parameters} more than once in a docstring!" + block = """ + module m + m.func + docstring summary + + these are the params: + {parameters} + these are the params again: + {parameters} + """ + self.expect_failure(block, err, lineno=7) + + def test_kind_defining_class(self): + function = self.parse_function(""" + module m + class m.C "PyObject *" "" + m.C.meth + cls: defining_class + """, signatures_in_block=3, function_index=2) + p = function.parameters['cls'] + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + def test_disallow_defining_class_at_module_level(self): + err = "A 'defining_class' parameter cannot be defined at module level." + block = """ + module m + m.func + cls: defining_class + """ + self.expect_failure(block, err, lineno=2) + + +class ClinicExternalTest(TestCase): + maxDiff = None + + def setUp(self): + save_restore_converters(self) + + def run_clinic(self, *args): + with ( + support.captured_stdout() as out, + support.captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + clinic.main(args) + return out.getvalue(), err.getvalue(), cm.exception.code + + def expect_success(self, *args): + out, err, code = self.run_clinic(*args) + if code != 0: + self.fail("\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") + return out + + def expect_failure(self, *args): + out, err, code = self.run_clinic(*args) + self.assertNotEqual(code, 0, f"Unexpected success: {args=}") + return out, err + + def test_external(self): + CLINIC_TEST = 'clinic.test.c' + source = support.findfile(CLINIC_TEST) + with open(source, encoding='utf-8') as f: + orig_contents = f.read() + + # Run clinic CLI and verify that it does not complain. + self.addCleanup(unlink, TESTFN) + out = self.expect_success("-f", "-o", TESTFN, source) + self.assertEqual(out, "") + + with open(TESTFN, encoding='utf-8') as f: + new_contents = f.read() + + self.assertEqual(new_contents, orig_contents) + + def test_no_change(self): + # bpo-42398: Test that the destination file is left unchanged if the + # content does not change. Moreover, check also that the file + # modification time does not change in this case. + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=da39a3ee5e6b4b0d]*/ + """) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(code) + pre_mtime = os.stat(fn).st_mtime_ns + self.expect_success(fn) + post_mtime = os.stat(fn).st_mtime_ns + # Don't change the file modification time + # if the content does not change + self.assertEqual(pre_mtime, post_mtime) + + def test_cli_force(self): + invalid_input = dedent(""" + /*[clinic input] + output preset block + module test + test.fn + a: int + [clinic start generated code]*/ + + const char *hand_edited = "output block is overwritten"; + /*[clinic end generated code: output=bogus input=bogus]*/ + """) + fail_msg = ( + "Checksum mismatch! Expected 'bogus', computed '2ed19'. " + "Suggested fix: remove all generated code including the end marker, " + "or use the '-f' option.\n" + ) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(invalid_input) + # First, run the CLI without -f and expect failure. + # Note, we cannot check the entire fail msg, because the path to + # the tmp file will change for every run. + _, err = self.expect_failure(fn) + self.assertEndsWith(err, fail_msg) + # Then, force regeneration; success expected. + out = self.expect_success("-f", fn) + self.assertEqual(out, "") + # Verify by checking the checksum. + checksum = ( + "/*[clinic end generated code: " + "output=a2957bc4d43a3c2f input=9543a8d2da235301]*/\n" + ) + with open(fn, encoding='utf-8') as f: + generated = f.read() + self.assertEndsWith(generated, checksum) + + def test_cli_make(self): + c_code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + py_code = "pass" + c_files = "file1.c", "file2.c" + py_files = "file1.py", "file2.py" + + def create_files(files, srcdir, code): + for fn in files: + path = os.path.join(srcdir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + with os_helper.temp_dir() as tmp_dir: + # add some folders, some C files and a Python file + create_files(c_files, tmp_dir, c_code) + create_files(py_files, tmp_dir, py_code) + + # create C files in externals/ dir + ext_path = os.path.join(tmp_dir, "externals") + with os_helper.temp_dir(path=ext_path) as externals: + create_files(c_files, externals, c_code) + + # run clinic in verbose mode with --make on tmpdir + out = self.expect_success("-v", "--make", "--srcdir", tmp_dir) + + # expect verbose mode to only mention the C files in tmp_dir + for filename in c_files: + with self.subTest(filename=filename): + path = os.path.join(tmp_dir, filename) + self.assertIn(path, out) + for filename in py_files: + with self.subTest(filename=filename): + path = os.path.join(tmp_dir, filename) + self.assertNotIn(path, out) + # don't expect C files from the externals dir + for filename in c_files: + with self.subTest(filename=filename): + path = os.path.join(ext_path, filename) + self.assertNotIn(path, out) + + def test_cli_make_exclude(self): + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + with os_helper.temp_dir(quiet=False) as tmp_dir: + # add some folders, some C files and a Python file + for fn in "file1.c", "file2.c", "file3.c", "file4.c": + path = os.path.join(tmp_dir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + # Run clinic in verbose mode with --make on tmpdir. + # Exclude file2.c and file3.c. + out = self.expect_success( + "-v", "--make", "--srcdir", tmp_dir, + "--exclude", os.path.join(tmp_dir, "file2.c"), + # The added ./ should be normalised away. + "--exclude", os.path.join(tmp_dir, "./file3.c"), + # Relative paths should also work. + "--exclude", "file4.c" + ) + + # expect verbose mode to only mention the C files in tmp_dir + self.assertIn("file1.c", out) + self.assertNotIn("file2.c", out) + self.assertNotIn("file3.c", out) + self.assertNotIn("file4.c", out) + + def test_cli_verbose(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("") + out = self.expect_success("-v", fn) + self.assertEqual(out.strip(), fn) + + @support.force_not_colorized + def test_cli_help(self): + out = self.expect_success("-h") + self.assertIn("usage: clinic.py", out) + + def test_cli_converters(self): + prelude = dedent(""" + Legacy converters: + B C D L O S U Y Z Z# + b c d f h i l p s s# s* u u# w* y y# y* z z# z* + + Converters: + """) + expected_converters = ( + "bool", + "byte", + "char", + "defining_class", + "double", + "fildes", + "float", + "int", + "long", + "long_long", + "object", + "Py_buffer", + "Py_complex", + "Py_ssize_t", + "Py_UNICODE", + "PyByteArrayObject", + "PyBytesObject", + "self", + "short", + "size_t", + "slice_index", + "str", + "uint16", + "uint32", + "uint64", + "uint8", + "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", + "unsigned_char", + "unsigned_int", + "unsigned_long", + "unsigned_long_long", + "unsigned_short", + ) + finale = dedent(""" + Return converters: + bool() + double() + float() + int() + long() + object() + Py_ssize_t() + size_t() + unsigned_int() + unsigned_long() + + All converters also accept (c_default=None, py_default=None, annotation=None). + All return converters also accept (py_default=None). + """) + out = self.expect_success("--converters") + # We cannot simply compare the output, because the repr of the *accept* + # param may change (it's a set, thus unordered). So, let's compare the + # start and end of the expected output, and then assert that the + # converters appear lined up in alphabetical order. + self.assertStartsWith(out, prelude) + self.assertEndsWith(out, finale) + + out = out.removeprefix(prelude) + out = out.removesuffix(finale) + lines = out.split("\n") + for converter, line in zip(expected_converters, lines): + line = line.lstrip() + with self.subTest(converter=converter): + self.assertStartsWith(line, converter) + + def test_cli_fail_converters_and_filename(self): + _, err = self.expect_failure("--converters", "test.c") + msg = "can't specify --converters and a filename at the same time" + self.assertIn(msg, err) + + def test_cli_fail_no_filename(self): + _, err = self.expect_failure() + self.assertIn("no input files", err) + + def test_cli_fail_output_and_multiple_files(self): + _, err = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") + msg = "error: can't use -o with multiple filenames" + self.assertIn(msg, err) + + def test_cli_fail_filename_or_output_and_make(self): + msg = "can't use -o or filenames with --make" + for opts in ("-o", "out.c"), ("filename.c",): + with self.subTest(opts=opts): + _, err = self.expect_failure("--make", *opts) + self.assertIn(msg, err) + + def test_cli_fail_make_without_srcdir(self): + _, err = self.expect_failure("--make", "--srcdir", "") + msg = "error: --srcdir must not be empty with --make" + self.assertIn(msg, err) + + def test_file_dest(self): + block = dedent(""" + /*[clinic input] + destination test new file {path}.h + output everything test + func + a: object + / + [clinic start generated code]*/ + """) + expected_checksum_line = ( + "/*[clinic end generated code: " + "output=da39a3ee5e6b4b0d input=b602ab8e173ac3bd]*/\n" + ) + expected_output = dedent("""\ + /*[clinic input] + preserve + [clinic start generated code]*/ + + PyDoc_VAR(func__doc__); + + PyDoc_STRVAR(func__doc__, + "func($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FUNC_METHODDEF \\ + {"func", (PyCFunction)func, METH_O, func__doc__}, + + static PyObject * + func(PyObject *module, PyObject *a) + /*[clinic end generated code: output=3dde2d13002165b9 input=a9049054013a1b77]*/ + """) + with os_helper.temp_dir() as tmp_dir: + in_fn = os.path.join(tmp_dir, "test.c") + out_fn = os.path.join(tmp_dir, "test.c.h") + with open(in_fn, "w", encoding="utf-8") as f: + f.write(block) + with open(out_fn, "w", encoding="utf-8") as f: + f.write("") # Write an empty output file! + # Clinic should complain about the empty output file. + _, err = self.expect_failure(in_fn) + expected_err = (f"Modified destination file {out_fn!r}; " + "not overwriting!") + self.assertIn(expected_err, err) + # Run clinic again, this time with the -f option. + _ = self.expect_success("-f", in_fn) + # Read back the generated output. + with open(in_fn, encoding="utf-8") as f: + data = f.read() + expected_block = f"{block}{expected_checksum_line}" + self.assertEqual(data, expected_block) + with open(out_fn, encoding="utf-8") as f: + data = f.read() + self.assertEqual(data, expected_output) + +try: + import _testclinic as ac_tester +except ImportError: + ac_tester = None + +@unittest.skipIf(ac_tester is None, "_testclinic is missing") +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def check_depr(self, regex, fn, /, *args, **kwds): + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_star(self, pnames, fn, /, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + regex = ( + fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " + fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " + fr"become( a)? keyword-only parameters? in Python 3\.14" + ) + self.check_depr(regex, fn, *args, **kwds) + + def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + pl = 's' if ' ' in pnames else '' + regex = ( + fr"Passing keyword argument{pl} {pnames} to " + fr"{re.escape(name)}\(\) is deprecated. Parameter{pl} {pnames} " + fr"will become positional-only in Python 3\.14." + ) + self.check_depr(regex, fn, *args, **kwds) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_bool_converter_c_default(self): + self.assertEqual(ac_tester.bool_converter_c_default(), (1, 0, -2, -3)) + self.assertEqual(ac_tester.bool_converter_c_default(False, True, False, True), + (0, 1, 0, 1)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if ac_tester is not None: + @repeat_fn(ac_tester.varpos, + ac_tester.varpos_array, + ac_tester.TestClass.varpos_no_fastcall, + ac_tester.TestClass.varpos_array_no_fastcall) + def test_varpos(self, fn): + # fn(*args) + self.assertEqual(fn(), ()) + self.assertEqual(fn(1, 2), (1, 2)) + + @repeat_fn(ac_tester.posonly_varpos, + ac_tester.posonly_varpos_array, + ac_tester.TestClass.posonly_varpos_no_fastcall, + ac_tester.TestClass.posonly_varpos_array_no_fastcall) + def test_posonly_varpos(self, fn): + # fn(a, b, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_req_opt_varpos, + ac_tester.posonly_req_opt_varpos_array, + ac_tester.TestClass.posonly_req_opt_varpos_no_fastcall, + ac_tester.TestClass.posonly_req_opt_varpos_array_no_fastcall) + def test_posonly_req_opt_varpos(self, fn): + # fn(a, b=False, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, a=1) + self.assertEqual(fn(1), (1, False, ())) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_poskw_varpos, + ac_tester.posonly_poskw_varpos_array, + ac_tester.TestClass.posonly_poskw_varpos_no_fastcall, + ac_tester.TestClass.posonly_poskw_varpos_array_no_fastcall) + def test_posonly_poskw_varpos(self, fn): + # fn(a, /, b, *args) + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, b=2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + self.assertRaises(TypeError, fn, b=4) + errmsg = re.escape("given by name ('b') and position (2)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, b=4) + + def test_poskw_varpos(self): + # fn(a, *args) + fn = ac_tester.poskw_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(a=1), (1, ())) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1), (1, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_poskw_varpos_kwonly_opt(self): + # fn(a, *args, b=False) + fn = ac_tester.poskw_varpos_kwonly_opt + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), True)) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) + self.assertEqual(fn(a=1), (1, (), False)) + self.assertEqual(fn(a=1, b=2), (1, (), True)) + + def test_poskw_varpos_kwonly_opt2(self): + # fn(a, *args, b=False, c=False) + fn = ac_tester.poskw_varpos_kwonly_opt2 + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), 2, False)) + self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) + self.assertEqual(fn(1, 2, 3), (1, (2, 3), False, False)) + self.assertEqual(fn(1, 2, 3, b=4), (1, (2, 3), 4, False)) + self.assertEqual(fn(1, 2, 3, b=4, c=5), (1, (2, 3), 4, 5)) + self.assertEqual(fn(a=1), (1, (), False, False)) + self.assertEqual(fn(a=1, b=2), (1, (), 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), (1, (), 2, 3)) + + def test_varpos_kwonly_opt(self): + # fn(*args, b=False) + fn = ac_tester.varpos_kwonly_opt + self.assertEqual(fn(), ((), False)) + self.assertEqual(fn(b=2), ((), 2)) + self.assertEqual(fn(1, b=2), ((1, ), 2)) + self.assertEqual(fn(1, 2, 3, 4), ((1, 2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_varpos_kwonly_req_opt(self): + fn = ac_tester.varpos_kwonly_req_opt + self.assertRaises(TypeError, fn) + self.assertEqual(fn(a=1), ((), 1, False, False)) + self.assertEqual(fn(a=1, b=2), ((), 1, 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), ((), 1, 2, 3)) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, a=2), ((1,), 2, False, False)) + self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) + self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) + + def test_gh_32092_oob(self): + ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) + + def test_gh_32092_kw_pass(self): + ac_tester.gh_32092_kw_pass(1, 2, 3) + + def test_gh_99233_refcount(self): + arg = '*A unique string is not referenced by anywhere else.*' + arg_refcount_origin = sys.getrefcount(arg) + ac_tester.gh_99233_refcount(arg) + arg_refcount_after = sys.getrefcount(arg) + self.assertEqual(arg_refcount_origin, arg_refcount_after) + + def test_gh_99240_double_free(self): + err = re.escape( + "gh_99240_double_free() argument 2 must be encoded string " + "without null bytes, not str" + ) + with self.assertRaisesRegex(TypeError, err): + ac_tester.gh_99240_double_free('a', '\0b') + + def test_null_or_tuple_for_varargs(self): + # fn(name, *constraints, covariant=False) + fn = ac_tester.null_or_tuple_for_varargs + # All of these should not crash: + self.assertEqual(fn('a'), ('a', (), False)) + self.assertEqual(fn('a', 1, 2, 3, covariant=True), ('a', (1, 2, 3), True)) + self.assertEqual(fn(name='a'), ('a', (), False)) + self.assertEqual(fn(name='a', covariant=True), ('a', (), True)) + self.assertEqual(fn(covariant=True, name='a'), ('a', (), True)) + + self.assertRaises(TypeError, fn, covariant=True) + errmsg = re.escape("given by name ('name') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, name='a') + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, name='a', covariant=True) + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, covariant=True, name='a') + + def test_cloned_func_exception_message(self): + incorrect_arg = -1 # f1() and f2() accept a single str + with self.assertRaisesRegex(TypeError, "clone_f1"): + ac_tester.clone_f1(incorrect_arg) + with self.assertRaisesRegex(TypeError, "clone_f2"): + ac_tester.clone_f2(incorrect_arg) + + def test_cloned_func_with_converter_exception_message(self): + for name in "clone_with_conv_f1", "clone_with_conv_f2": + with self.subTest(name=name): + func = getattr(ac_tester, name) + self.assertEqual(func(), name) + + def test_get_defining_class(self): + obj = ac_tester.TestClass() + meth = obj.get_defining_class + self.assertIs(obj.get_defining_class(), ac_tester.TestClass) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_get_defining_class_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.get_defining_class + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + self.assertIs(pyobject_vectorcall(meth, (), ()), ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + + def test_get_defining_class_arg(self): + obj = ac_tester.TestClass() + self.assertEqual(obj.get_defining_class_arg("arg"), + (ac_tester.TestClass, "arg")) + self.assertEqual(obj.get_defining_class_arg(arg=123), + (ac_tester.TestClass, 123)) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass, arg="arg") + + # wrong number of arguments + with self.assertRaises(TypeError): + obj.get_defining_class_arg() + with self.assertRaises(TypeError): + obj.get_defining_class_arg("arg1", "arg2") + + def test_defclass_varpos(self): + # fn(*args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_varpos + self.assertEqual(fn(), (cls, ())) + self.assertEqual(fn(1, 2), (cls, (1, 2))) + fn = cls.defclass_varpos + self.assertRaises(TypeError, fn) + self.assertEqual(fn(obj), (cls, ())) + self.assertEqual(fn(obj, 1, 2), (cls, (1, 2))) + + def test_defclass_posonly_varpos(self): + # fn(a, b, /, *args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_posonly_varpos + errmsg = 'takes at least 2 positional arguments' + self.assertRaisesRegex(TypeError, errmsg, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, 1) + self.assertEqual(fn(1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (cls, 1, 2, (3, 4))) + fn = cls.defclass_posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, obj) + self.assertRaisesRegex(TypeError, errmsg, fn, obj, 1) + self.assertEqual(fn(obj, 1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(obj, 1, 2, 3, 4), (cls, 1, 2, (3, 4))) + + def test_depr_star_new(self): + cls = ac_tester.DeprStarNew + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) + + def test_depr_star_new_cloned(self): + fn = ac_tester.DeprStarNew().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarNew.cloned') + + def test_depr_star_init(self): + cls = ac_tester.DeprStarInit + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) + + def test_depr_star_init_cloned(self): + fn = ac_tester.DeprStarInit().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarInit.cloned') + + def test_depr_star_init_noinline(self): + cls = ac_tester.DeprStarInitNoInline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_kwd_new(self): + cls = ac_tester.DeprKwdNew + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init(self): + cls = ac_tester.DeprKwdInit + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init_noinline(self): + cls = ac_tester.DeprKwdInitNoInline + cls = ac_tester.depr_star_noinline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_star_pos0_len1(self): + fn = ac_tester.depr_star_pos0_len1 + fn(a=None) + self.check_depr_star("'a'", fn, "a") + + def test_depr_star_pos0_len2(self): + fn = ac_tester.depr_star_pos0_len2 + fn(a=0, b=0) + check = partial(self.check_depr_star, "'a' and 'b'", fn) + check("a", b=0) + check("a", "b") + + def test_depr_star_pos0_len3_with_kwd(self): + fn = ac_tester.depr_star_pos0_len3_with_kwd + fn(a=0, b=0, c=0, d=0) + check = partial(self.check_depr_star, "'a', 'b' and 'c'", fn) + check("a", b=0, c=0, d=0) + check("a", "b", c=0, d=0) + check("a", "b", "c", d=0) + + def test_depr_star_pos1_len1_opt(self): + fn = ac_tester.depr_star_pos1_len1_opt + fn(a=0, b=0) + fn("a", b=0) + fn(a=0) # b is optional + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len1(self): + fn = ac_tester.depr_star_pos1_len1 + fn(a=0, b=0) + fn("a", b=0) + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len2_with_kwd(self): + fn = ac_tester.depr_star_pos1_len2_with_kwd + fn(a=0, b=0, c=0, d=0), + fn("a", b=0, c=0, d=0), + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b", c=0, d=0), + check("a", "b", "c", d=0), + + def test_depr_star_pos2_len1(self): + fn = ac_tester.depr_star_pos2_len1 + fn(a=0, b=0, c=0) + fn("a", b=0, c=0) + fn("a", "b", c=0) + check = partial(self.check_depr_star, "'c'", fn) + check("a", "b", "c") + + def test_depr_star_pos2_len2(self): + fn = ac_tester.depr_star_pos2_len2 + fn(a=0, b=0, c=0, d=0) + fn("a", b=0, c=0, d=0) + fn("a", "b", c=0, d=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0) + check("a", "b", "c", "d") + + def test_depr_star_pos2_len2_with_kwd(self): + fn = ac_tester.depr_star_pos2_len2_with_kwd + fn(a=0, b=0, c=0, d=0, e=0) + fn("a", b=0, c=0, d=0, e=0) + fn("a", "b", c=0, d=0, e=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0, e=0) + check("a", "b", "c", "d", e=0) + + def test_depr_star_noinline(self): + fn = ac_tester.depr_star_noinline + self.assertRaises(TypeError, fn, "a") + fn(a="a", b="b") + fn(a="a", b="b", c="c") + fn("a", b="b") + fn("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + + def test_depr_star_multi(self): + fn = ac_tester.depr_star_multi + self.assertRaises(TypeError, fn, "a") + fn("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + errmsg = ( + "Passing more than 1 positional argument to depr_star_multi() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python 3.16. " + "Parameters 'c' and 'd' will become keyword-only parameters in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become keyword-only parameters in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", "g", h="h") + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g", "h") + + def test_depr_kwd_required_1(self): + fn = ac_tester.depr_kwd_required_1 + fn("a", "b") + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_required_2(self): + fn = ac_tester.depr_kwd_required_2 + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_1(self): + fn = ac_tester.depr_kwd_optional_1 + fn("a") + fn("a", "b") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_optional_2(self): + fn = ac_tester.depr_kwd_optional_2 + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", c="c") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_3(self): + fn = ac_tester.depr_kwd_optional_3 + fn() + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'a', 'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b") + check(a="a") + + def test_depr_kwd_required_optional(self): + fn = ac_tester.depr_kwd_required_optional + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_noinline(self): + fn = ac_tester.depr_kwd_noinline + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_multi(self): + fn = ac_tester.depr_kwd_multi + fn("a", "b", "c", "d", "e", "f", "g", h="h") + errmsg = ( + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to depr_kwd_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameters 'c' and 'd' will become positional-only in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become positional-only in Python 3.16.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + + def test_depr_multi(self): + fn = ac_tester.depr_multi + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g") + errmsg = ( + "Passing more than 4 positional arguments to depr_multi() is deprecated. " + "Parameter 'e' will become a keyword-only parameter in Python 3.15. " + "Parameter 'f' will become a keyword-only parameter in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g") + check("a", "b", "c", "d", "e", f="f", g="g") + fn("a", "b", "c", "d", e="e", f="f", g="g") + fn("a", "b", "c", d="d", e="e", f="f", g="g") + errmsg = ( + "Passing keyword arguments 'b' and 'c' to depr_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameter 'c' will become positional-only in Python 3.15.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g") + check("a", b="b", c="c", d="d", e="e", f="f", g="g") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") + + +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + +try: + import _testclinic_limited +except ImportError: + _testclinic_limited = None + +@unittest.skipIf(_testclinic_limited is None, "_testclinic_limited is missing") +class LimitedCAPIFunctionalTest(unittest.TestCase): + locals().update((name, getattr(_testclinic_limited, name)) + for name in dir(_testclinic_limited) if name.startswith('test_')) + + def test_my_int_func(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func() + self.assertEqual(_testclinic_limited.my_int_func(3), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func(1.0) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func("xyz") + + def test_my_int_sum(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum() + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1) + self.assertEqual(_testclinic_limited.my_int_sum(1, 2), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1.0, 2) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1, "str") + + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) + + +class PermutationTests(unittest.TestCase): + """Test permutation support functions.""" + + def test_permute_left_option_groups(self): + expected = ( + (), + (3,), + (2, 3), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(permute_left_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_right_option_groups(self): + expected = ( + (), + (1,), + (1, 2), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(permute_right_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_optional_groups(self): + empty = { + "left": (), "required": (), "right": (), + "expected": ((),), + } + noleft1 = { + "left": (), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("b", "c"), + ), + } + noleft2 = { + "left": (), "required": ("b", "c",), "right": ("d",), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ), + } + noleft3 = { + "left": (), "required": ("b", "c",), "right": ("d", "e"), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ("b", "c", "d", "e"), + ), + } + noright1 = { + "left": ("a",), "required": ("b",), "right": (), + "expected": ( + ("b",), + ("a", "b"), + ), + } + noright2 = { + "left": ("a",), "required": ("b", "c"), "right": (), + "expected": ( + ("b", "c"), + ("a", "b", "c"), + ), + } + noright3 = { + "left": ("a", "b"), "required": ("c",), "right": (), + "expected": ( + ("c",), + ("b", "c"), + ("a", "b", "c"), + ), + } + leftandright1 = { + "left": ("a",), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("a", "b"), # Prefer left. + ("a", "b", "c"), + ), + } + leftandright2 = { + "left": ("a", "b"), "required": ("c", "d"), "right": ("e", "f"), + "expected": ( + ("c", "d"), + ("b", "c", "d"), # Prefer left. + ("a", "b", "c", "d"), # Prefer left. + ("a", "b", "c", "d", "e"), + ("a", "b", "c", "d", "e", "f"), + ), + } + dataset = ( + empty, + noleft1, noleft2, noleft3, + noright1, noright2, noright3, + leftandright1, leftandright2, + ) + for params in dataset: + with self.subTest(**params): + left, required, right, expected = params.values() + permutations = permute_optional_groups(left, required, right) + actual = tuple(permutations) + self.assertEqual(actual, expected) + + +class FormatHelperTests(unittest.TestCase): + + def test_strip_leading_and_trailing_blank_lines(self): + dataset = ( + # Input lines, expected output. + ("a\nb", "a\nb"), + ("a\nb\n", "a\nb"), + ("a\nb ", "a\nb"), + ("\na\nb\n\n", "a\nb"), + ("\n\na\nb\n\n", "a\nb"), + ("\n\na\n\nb\n\n", "a\n\nb"), + # Note, leading whitespace is preserved: + (" a\nb", " a\nb"), + (" a\nb ", " a\nb"), + (" \n \n a\nb \n \n ", " a\nb"), + ) + for lines, expected in dataset: + with self.subTest(lines=lines, expected=expected): + out = libclinic.normalize_snippet(lines) + self.assertEqual(out, expected) + + def test_normalize_snippet(self): + snippet = """ + one + two + three + """ + + # Expected outputs: + zero_indent = ( + "one\n" + "two\n" + "three" + ) + four_indent = ( + " one\n" + " two\n" + " three" + ) + eight_indent = ( + " one\n" + " two\n" + " three" + ) + expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} + for indent, expected in expected_outputs.items(): + with self.subTest(indent=indent): + actual = libclinic.normalize_snippet(snippet, indent=indent) + self.assertEqual(actual, expected) + + def test_escaped_docstring(self): + dataset = ( + # input, expected + (r"abc", r'"abc"'), + (r"\abc", r'"\\abc"'), + (r"\a\bc", r'"\\a\\bc"'), + (r"\a\\bc", r'"\\a\\\\bc"'), + (r'"abc"', r'"\"abc\""'), + (r"'a'", r'"\'a\'"'), + ) + for line, expected in dataset: + with self.subTest(line=line, expected=expected): + out = libclinic.docstring_for_c_string(line) + self.assertEqual(out, expected) + + def test_format_escape(self): + line = "{}, {a}" + expected = "{{}}, {{a}}" + out = libclinic.format_escape(line) + self.assertEqual(out, expected) + + def test_c_bytes_repr(self): + c_bytes_repr = libclinic.c_bytes_repr + self.assertEqual(c_bytes_repr(b''), '""') + self.assertEqual(c_bytes_repr(b'abc'), '"abc"') + self.assertEqual(c_bytes_repr(b'\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_bytes_repr(b' \0\x7f'), r'" \000\177"') + self.assertEqual(c_bytes_repr(b'"'), r'"\""') + self.assertEqual(c_bytes_repr(b"'"), r'''"'"''') + self.assertEqual(c_bytes_repr(b'\\'), r'"\\"') + self.assertEqual(c_bytes_repr(b'??/'), r'"?\?/"') + self.assertEqual(c_bytes_repr(b'???/'), r'"?\?\?/"') + self.assertEqual(c_bytes_repr(b'/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_bytes_repr(b'\xa0'), r'"\240"') + self.assertEqual(c_bytes_repr(b'\xff'), r'"\377"') + + def test_c_str_repr(self): + c_str_repr = libclinic.c_str_repr + self.assertEqual(c_str_repr(''), '""') + self.assertEqual(c_str_repr('abc'), '"abc"') + self.assertEqual(c_str_repr('\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_str_repr(' \0\x7f'), r'" \000\177"') + self.assertEqual(c_str_repr('"'), r'"\""') + self.assertEqual(c_str_repr("'"), r'''"'"''') + self.assertEqual(c_str_repr('\\'), r'"\\"') + self.assertEqual(c_str_repr('??/'), r'"?\?/"') + self.assertEqual(c_str_repr('???/'), r'"?\?\?/"') + self.assertEqual(c_str_repr('/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_str_repr('\xa0'), r'"\u00a0"') + self.assertEqual(c_str_repr('\xff'), r'"\u00ff"') + self.assertEqual(c_str_repr('\u20ac'), r'"\u20ac"') + self.assertEqual(c_str_repr('\U0001f40d'), r'"\U0001f40d"') + + def test_c_unichar_repr(self): + c_unichar_repr = libclinic.c_unichar_repr + self.assertEqual(c_unichar_repr('a'), "'a'") + self.assertEqual(c_unichar_repr('\n'), r"'\n'") + self.assertEqual(c_unichar_repr('\b'), r"'\b'") + self.assertEqual(c_unichar_repr('\0'), '0') + self.assertEqual(c_unichar_repr('\1'), '0x01') + self.assertEqual(c_unichar_repr('\x7f'), '0x7f') + self.assertEqual(c_unichar_repr(' '), "' '") + self.assertEqual(c_unichar_repr('"'), """'"'""") + self.assertEqual(c_unichar_repr("'"), r"'\''") + self.assertEqual(c_unichar_repr('\\'), r"'\\'") + self.assertEqual(c_unichar_repr('?'), "'?'") + self.assertEqual(c_unichar_repr('\xa0'), '0xa0') + self.assertEqual(c_unichar_repr('\xff'), '0xff') + self.assertEqual(c_unichar_repr('\u20ac'), '0x20ac') + self.assertEqual(c_unichar_repr('\U0001f40d'), '0x1f40d') + + def test_indent_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "barone\n" + "bartwo" + ) + out = libclinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "bar\n" + "barone\n" + "bartwo\n" + "" + ) + out = libclinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + def test_suffix_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(libclinic.suffix_all_lines("", suffix="foo"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "onefoo\n" + "twofoo" + ) + out = libclinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "foo\n" + "onefoo\n" + "twofoo\n" + "" + ) + out = libclinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + +class ClinicReprTests(unittest.TestCase): + def test_Block_repr(self): + block = Block("foo") + expected_repr = "" + self.assertEqual(repr(block), expected_repr) + + block2 = Block("bar", "baz", [], "eggs", "spam") + expected_repr_2 = "" + self.assertEqual(repr(block2), expected_repr_2) + + block3 = Block( + input="longboi_" * 100, + dsl_name="wow_so_long", + signatures=[], + output="very_long_" * 100, + indent="" + ) + expected_repr_3 = ( + "" + ) + self.assertEqual(repr(block3), expected_repr_3) + + def test_Destination_repr(self): + c = _make_clinic() + + destination = Destination( + "foo", type="file", clinic=c, args=("eggs",) + ) + self.assertEqual( + repr(destination), "" + ) + + destination2 = Destination("bar", type="buffer", clinic=c) + self.assertEqual(repr(destination2), "") + + def test_Module_repr(self): + module = Module("foo", _make_clinic()) + self.assertRegex(repr(module), r"") + + def test_Class_repr(self): + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + self.assertRegex(repr(cls), r"") + + def test_FunctionKind_repr(self): + self.assertEqual( + repr(FunctionKind.CLASS_METHOD), "" + ) + + def test_Function_and_Parameter_reprs(self): + function = Function( + name='foo', + module=_make_clinic(), + cls=None, + c_basename=None, + full_name='foofoo', + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, + coexist=False + ) + self.assertEqual(repr(function), "") + + converter = self_converter('bar', 'bar', function) + parameter = Parameter( + "bar", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + function=function, + converter=converter + ) + self.assertEqual(repr(parameter), "") + + def test_Monitor_repr(self): + monitor = libclinic.cpp.Monitor("test.c") + self.assertRegex(repr(monitor), r"") + + monitor.line_number = 42 + monitor.stack.append(("token1", "condition1")) + self.assertRegex( + repr(monitor), r"" + ) + + monitor.stack.append(("token2", "condition2")) + self.assertRegex( + repr(monitor), + r"" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py new file mode 100644 index 00000000000..fc34ac2fdc9 --- /dev/null +++ b/Lib/test/test_generated_cases.py @@ -0,0 +1,2074 @@ +import contextlib +import os +import re +import sys +import tempfile +import unittest + +from io import StringIO +from test import support +from test import test_tools + + +def skip_if_different_mount_drives(): + if sys.platform != "win32": + return + ROOT = os.path.dirname(os.path.dirname(__file__)) + root_drive = os.path.splitroot(ROOT)[0] + cwd_drive = os.path.splitroot(os.getcwd())[0] + if root_drive != cwd_drive: + # May raise ValueError if ROOT and the current working + # different have different mount drives (on Windows). + raise unittest.SkipTest( + f"the current working directory and the Python source code " + f"directory have different mount drives " + f"({cwd_drive} and {root_drive})" + ) + + +skip_if_different_mount_drives() + + +test_tools.skip_if_missing("cases_generator") +with test_tools.imports_under_tool("cases_generator"): + from analyzer import analyze_forest, StackItem + from cwriter import CWriter + import parser + from stack import Local, Stack + import tier1_generator + import opcode_metadata_generator + import optimizer_generator + + +def handle_stderr(): + if support.verbose > 1: + return contextlib.nullcontext() + else: + return support.captured_stderr() + + +def parse_src(src): + p = parser.Parser(src, "test.c") + nodes = [] + while node := p.definition(): + nodes.append(node) + return nodes + + +class TestEffects(unittest.TestCase): + def test_effect_sizes(self): + stack = Stack() + inputs = [ + x := StackItem("x", None, "1"), + y := StackItem("y", None, "oparg"), + z := StackItem("z", None, "oparg*2"), + ] + outputs = [ + StackItem("x", None, "1"), + StackItem("b", None, "oparg*4"), + StackItem("c", None, "1"), + ] + null = CWriter.null() + stack.pop(z, null) + stack.pop(y, null) + stack.pop(x, null) + for out in outputs: + stack.push(Local.undefined(out)) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") + self.assertEqual(stack.physical_sp.to_c(), "0") + self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4") + + +class TestGeneratedCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + tier1_generator.generate_tier1_from_files( + [self.temp_input_filename], self.temp_output_filename, False + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions.strip() + "\n\n " + labels.strip() + + self.assertEqual(actual.strip(), expected.strip()) + + def test_inst_no_args(self): + input = """ + inst(OP, (--)) { + SPAM(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_pop(self): + input = """ + inst(OP, (value --)) { + SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + SPAM(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push(self): + input = """ + inst(OP, (-- res)) { + res = SPAM(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef res; + res = SPAM(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push_one_pop(self): + input = """ + inst(OP, (value -- res)) { + res = SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + res = SPAM(value); + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_binary_op(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_overlap(self): + input = """ + inst(OP, (left, right -- left, result)) { + result = SPAM(left, right); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + result = SPAM(left, right); + stack_pointer[-1] = result; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_predictions(self): + input = """ + inst(OP1, (arg -- res)) { + DEAD(arg); + res = Py_None; + } + inst(OP3, (arg -- res)) { + DEAD(arg); + DEOPT_IF(xxx); + res = Py_None; + } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + PREDICTED_OP1:; + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(OP3) { + #if Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + if (xxx) { + UPDATE_MISS_STATS(OP1); + assert(_PyOpcode_Deopt[opcode] == (OP1)); + JUMP_TO_PREDICTED(OP1); + } + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + DEAD(arg); + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + DEAD(arg); + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + #if Py_TAIL_CALL_INTERP + int opcode = A; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(B) { + #if Py_TAIL_CALL_INTERP + int opcode = B; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_error_if_plain(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_plain_with_comment(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); // Comment is ok + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop(self): + input = """ + inst(OP, (left, right -- res)) { + SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + res = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + res = 0; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_cache_effect(self): + input = """ + inst(OP, (counter/1, extra/2, value --)) { + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_suppress_dispatch(self): + input = """ + label(somewhere) { + } + + inst(OP, (--)) { + goto somewhere; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + JUMP_TO_LABEL(somewhere); + } + + LABEL(somewhere) + { + } + """ + self.run_cases_test(input, output) + + def test_macro_instruction(self): + input = """ + inst(OP1, (counter/1, left, right -- left, right)) { + op1(left, right); + } + op(OP2, (extra/2, arg2, left, right -- res)) { + res = op2(arg2, left, right); + INPUTS_DEAD(); + } + macro(OP) = OP1 + cache/2 + OP2; + inst(OP3, (unused/5, arg2, left, right -- res)) { + res = op3(arg2, left, right); + INPUTS_DEAD(); + } + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); + PREDICTED_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 2 cache entries */ + // OP2 + { + arg2 = stack_pointer[-3]; + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); + _PyStackRef left; + _PyStackRef right; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(OP3) { + #if Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 5 cache entries */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_caches(self): + input = """ + inst(OP, (unused/1, unused/2 --)) { + body; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + body; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_no_flags(self): + input = """ + pseudo(OP, (in -- out1, out2)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_with_flags(self): + input = """ + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + #if Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_array_input(self): + input = """ + inst(OP, (below, values[oparg*2], above --)) { + SPAM(values, oparg); + DEAD(below); + DEAD(values); + DEAD(above); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + above = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg*2]; + below = stack_pointer[-2 - oparg*2]; + SPAM(values, oparg); + stack_pointer += -2 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_output(self): + input = """ + inst(OP, (unused, unused -- below, values[oparg*3], above)) { + SPAM(values, oparg); + below = 0; + above = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-1]; + SPAM(values, oparg); + below = 0; + above = 0; + stack_pointer[-2] = below; + stack_pointer[-1 + oparg*3] = above; + stack_pointer += oparg*3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input_output(self): + input = """ + inst(OP, (values[oparg] -- values[oparg], above)) { + SPAM(values, oparg); + above = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-oparg]; + SPAM(values, oparg); + above = 0; + stack_pointer[0] = above; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_error_if(self): + input = """ + inst(OP, (extra, values[oparg] --)) { + DEAD(extra); + DEAD(values); + ERROR_IF(oparg == 0); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef extra; + _PyStackRef *values; + values = &stack_pointer[-oparg]; + extra = stack_pointer[-1 - oparg]; + if (oparg == 0) { + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_macro_push_push(self): + input = """ + op(A, (-- val1)) { + val1 = SPAM(); + } + op(B, (-- val2)) { + val2 = SPAM(); + } + macro(M) = A + B; + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + _PyStackRef val1; + _PyStackRef val2; + // A + { + val1 = SPAM(); + } + // B + { + val2 = SPAM(); + } + stack_pointer[0] = val1; + stack_pointer[1] = val2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_inst(self): + input = """ + inst(OP, (--)) { + spam; + } + override inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_op(self): + input = """ + op(OP, (--)) { + spam; + } + macro(M) = OP; + override op(OP, (--)) { + ham; + } + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_inst(self): + input = """ + pure inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_op(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + input = """ + pure register specializing op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + self.run_cases_test(input, output) + + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pointer_to_stackref(self): + input = """ + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; + stack_pointer[-1] = out; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + #if Py_TAIL_CALL_INTERP + int opcode = INST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + USE(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef y; + // FIRST + { + w = stack_pointer[-1]; + USE(w); + } + // SECOND + { + } + // THIRD + { + y = w; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + USE(x); + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + { + x = stack_pointer[-1]; + USE(x); + } + // THIRD + { + y = x; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + USE(a, b); + INPUTS_DEAD(); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + // SECOND + { + USE(a, b); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + DEAD(x); + b = y; + DEAD(y); + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + INPUTS_DEAD(); // Mark j and k as used + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + // FIRST + { + y = stack_pointer[-1]; + x = stack_pointer[-2]; + a = x; + b = y; + } + // SECOND + { + } + // THIRD + { + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_push_then_error(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a -- a, b)) { + b = 1; + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 1; + } + // SECOND + { + b = 1; + if (cond) { + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + } + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_true(self): + + input = """ + inst(OP1, ( --)) { + ERROR_IF(true); + } + inst(OP2, ( --)) { + ERROR_IF(1); + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + JUMP_TO_LABEL(error); + } + + TARGET(OP2) { + #if Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + JUMP_TO_LABEL(error); + } + """ + self.run_cases_test(input, output) + + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + code(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + code(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_save_reload_paired(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_instruction_size_macro(self): + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + """ + + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + frame->return_offset = 1u ; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + # Two instructions of different sizes referencing the same + # uop containing the `INSTRUCTION_SIZE` macro is not allowed. + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + macro(OP2) = unused/1 + OP; + """ + + output = "" # No output needed as this should raise an error. + with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): + self.run_cases_test(input, output) + + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pystackref_frompyobject_new_next_to_cmacro(self): + input = """ + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_no_escaping_calls_in_branching_macros(self): + + input = """ + inst(OP, ( -- )) { + DEOPT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + EXIT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + ERROR_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_kill_in_wrong_order(self): + input = """ + inst(OP, (a, b -- c)) { + c = b; + PyStackRef_CLOSE(a); + PyStackRef_CLOSE(b); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_complex_label(self): + input = """ + label(other_label) { + } + + label(other_label2) { + } + + label(my_label) { + // Comment + do_thing(); + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + LABEL(other_label) + { + } + + LABEL(other_label2) + { + } + + LABEL(my_label) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (complex) { + JUMP_TO_LABEL(other_label); + } + JUMP_TO_LABEL(other_label2); + } + """ + self.run_cases_test(input, output) + + def test_spilled_label(self): + input = """ + spilled label(one) { + RELOAD_STACK(); + goto two; + } + + label(two) { + SAVE_STACK(); + goto one; + } + """ + + output = """ + LABEL(one) + { + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(two); + } + + LABEL(two) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(one); + } + """ + self.run_cases_test(input, output) + + + def test_incorrect_spills(self): + input1 = """ + spilled label(one) { + goto two; + } + + label(two) { + } + """ + + input2 = """ + spilled label(one) { + } + + label(two) { + goto one; + } + """ + with self.assertRaisesRegex(SyntaxError, ".*reload.*"): + self.run_cases_test(input1, "") + with self.assertRaisesRegex(SyntaxError, ".*spill.*"): + self.run_cases_test(input2, "") + + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_1; + } + """ + + output = """ + LABEL(my_label_1) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing1(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_2); + } + + LABEL(my_label_2) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing2(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_1); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_live_inputs(self): + input = """ + inst(OP, (in -- in)) { + in = 0; + } + """ + + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + in = 0; + stack_pointer[-1] = in; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_dead_inputs(self): + input = """ + inst(OP, (in -- )) { + temp = use(in); + DEAD(in); + in = temp; + PyStackRef_CLOSE(in); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + temp = use(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + in = temp; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + JitOptSymbol *arg1; + JitOptSymbol *out; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + out = NULL; + } + """ + output = """ + case OP: { + JitOptSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptSymbol *out; + out = NULL; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py new file mode 100644 index 00000000000..b32db4426f2 --- /dev/null +++ b/Lib/test/test_peg_generator/__init__.py @@ -0,0 +1,12 @@ +import os.path +from test import support +from test.support import load_package_tests + + +# Creating a virtual environment and building C extensions is slow +support.requires('cpu') + + +# Load all tests in package +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_peg_generator/__main__.py b/Lib/test/test_peg_generator/__main__.py new file mode 100644 index 00000000000..1fab1fddb57 --- /dev/null +++ b/Lib/test/test_peg_generator/__main__.py @@ -0,0 +1,4 @@ +import unittest +from . import load_tests + +unittest.main() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py new file mode 100644 index 00000000000..aa01a9b8f7e --- /dev/null +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -0,0 +1,523 @@ +import contextlib +import subprocess +import sysconfig +import textwrap +import unittest +import os +import shutil +import tempfile +from pathlib import Path + +from test import test_tools +from test import support +from test.support import os_helper, import_helper +from test.support.script_helper import assert_python_ok + +if support.check_cflags_pgo(): + raise unittest.SkipTest("peg_generator test disabled under PGO build") + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import ( + parse_string, + generate_parser_c_extension, + generate_c_parser_source, + ) + + +TEST_TEMPLATE = """ +tmp_dir = {extension_path!r} + +import ast +import traceback +import sys +import unittest + +from test import test_tools +with test_tools.imports_under_tool("peg_generator"): + from pegen.ast_dump import ast_dump + +sys.path.insert(0, tmp_dir) +import parse + +class Tests(unittest.TestCase): + + def check_input_strings_for_grammar( + self, + valid_cases = (), + invalid_cases = (), + ): + if valid_cases: + for case in valid_cases: + parse.parse_string(case, mode=0) + + if invalid_cases: + for case in invalid_cases: + with self.assertRaises(SyntaxError): + parse.parse_string(case, mode=0) + + def verify_ast_generation(self, stmt): + expected_ast = ast.parse(stmt) + actual_ast = parse.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + + def test_parse(self): + {test_source} + +unittest.main() +""" + + +@support.requires_subprocess() +class TestCParser(unittest.TestCase): + + _has_run = False + + @classmethod + def setUpClass(cls): + if cls._has_run: + # Since gh-104798 (Use setuptools in peg-generator and reenable + # tests), this test case has been producing ref leaks. Initial + # debugging points to bug(s) in setuptools and/or importlib. + # See gh-105063 for more info. + raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks") + cls._has_run = True + + # When running under regtest, a separate tempdir is used + # as the current directory and watched for left-overs. + # Reusing that as the base for temporary directories + # ensures everything is cleaned up properly and + # cleans up afterwards if not (with warnings). + cls.tmp_base = os.getcwd() + if os.path.samefile(cls.tmp_base, os_helper.SAVEDCWD): + cls.tmp_base = None + # Create a directory for the reuseable static library part of + # the pegen extension build process. This greatly reduces the + # runtime overhead of spawning compiler processes. + cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) + cls.addClassCleanup(shutil.rmtree, cls.library_dir) + + with contextlib.ExitStack() as stack: + python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) + sitepackages = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + cls.addClassCleanup(stack.pop_all().close) + + @support.requires_venv_with_pip() + def setUp(self): + self._backup_config_vars = dict(sysconfig._CONFIG_VARS) + cmd = support.missing_compiler_executable() + if cmd is not None: + self.skipTest("The %r command is not found" % cmd) + self.old_cwd = os.getcwd() + self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base) + self.enterContext(os_helper.change_cwd(self.tmp_path)) + + def tearDown(self): + os.chdir(self.old_cwd) + shutil.rmtree(self.tmp_path) + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(self._backup_config_vars) + + def build_extension(self, grammar_source): + grammar = parse_string(grammar_source, GrammarParser) + # Because setUp() already changes the current directory to the + # temporary path, use a relative path here to prevent excessive + # path lengths when compiling. + generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir) + + def run_test(self, grammar_source, test_source): + self.build_extension(grammar_source) + test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ") + assert_python_ok( + "-c", + TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source), + ) + + def test_c_parser(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) } + stmt[stmt_ty]: a=expr_stmt { a } + expr_stmt[stmt_ty]: a=expression NEWLINE { _PyAST_Expr(a, EXTRA) } + expression[expr_ty]: ( l=expression '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) } + | l=expression '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) } + | t=term { t } + ) + term[expr_ty]: ( l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) } + | f=factor { f } + ) + factor[expr_ty]: ('(' e=expression ')' { e } + | a=atom { a } + ) + atom[expr_ty]: ( n=NAME { n } + | n=NUMBER { n } + | s=STRING { s } + ) + """ + test_source = """ + expressions = [ + "4+5", + "4-5", + "4*5", + "1+4*5", + "1+4/5", + "(1+1) + (1+1)", + "(1+1) - (1+1)", + "(1+1) * (1+1)", + "(1+1) / (1+1)", + ] + + for expr in expressions: + the_ast = parse.parse_string(expr, mode=1) + expected_ast = ast.parse(expr) + self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast)) + """ + self.run_test(grammar_source, test_source) + + def test_lookahead(self) -> None: + grammar_source = """ + start: NAME &NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + test_source = """ + valid_cases = ["foo bar"] + invalid_cases = ["foo 34"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_negative_lookahead(self) -> None: + grammar_source = """ + start: NAME !NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + test_source = """ + valid_cases = ["foo 34"] + invalid_cases = ["foo bar"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_cut(self) -> None: + grammar_source = """ + start: X ~ Y Z | X Q S + X: 'x' + Y: 'y' + Z: 'z' + Q: 'q' + S: 's' + """ + test_source = """ + valid_cases = ["x y z"] + invalid_cases = ["x q s"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_gather(self) -> None: + grammar_source = """ + start: ';'.pass_stmt+ NEWLINE + pass_stmt: 'pass' + """ + test_source = """ + valid_cases = ["pass", "pass; pass"] + invalid_cases = ["pass;", "pass; pass;"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_left_recursion(self) -> None: + grammar_source = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + """ + test_source = """ + valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_advanced_left_recursive(self) -> None: + grammar_source = """ + start: NUMBER | sign start + sign: ['-'] + """ + test_source = """ + valid_cases = ["23", "-34"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_mutually_left_recursive(self) -> None: + grammar_source = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + test_source = """ + valid_cases = ["B E", "D A C A E"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_nasty_mutually_left_recursive(self) -> None: + grammar_source = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + test_source = """ + valid_cases = ["x ="] + invalid_cases = ["x - + ="] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_return_stmt_noexpr_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: return_stmt + return_stmt[stmt_ty]: a='return' NEWLINE { _PyAST_Return(NULL, EXTRA) } + """ + test_source = """ + stmt = "return" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_gather_action_ast(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=';'.pass_stmt+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA)} + """ + test_source = """ + stmt = "pass; pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_pass_stmt_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: pass_stmt + pass_stmt[stmt_ty]: a='pass' NEWLINE { _PyAST_Pass(EXTRA) } + """ + test_source = """ + stmt = "pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_if_stmt_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } + statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmt + + simple_stmt[asdl_stmt_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE { + (asdl_stmt_seq*)_PyPegen_seq_insert_in_front(p, a, b) } + further_small_stmt[stmt_ty]: ';' a=small_stmt { a } + + block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a } + + compound_stmt: if_stmt + + if_stmt: 'if' a=full_expression ':' b=block { _PyAST_If(a, b, NULL, EXTRA) } + + small_stmt[stmt_ty]: pass_stmt + + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) } + + full_expression: NAME + """ + test_source = """ + stmt = "pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_same_name_different_types(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=import_from+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena)} + import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from { + _PyAST_ImportFrom(c->v.Name.id, d, 0, EXTRA) } + | a='from' '.' 'import' c=import_as_names_from { + _PyAST_ImportFrom(NULL, c, 1, EXTRA) } + ) + simple_name[expr_ty]: NAME + import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a } + import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) } + """ + test_source = """ + for stmt in ("from a import b as c", "from . import a as b"): + expected_ast = ast.parse(stmt) + actual_ast = parse.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + """ + self.run_test(grammar_source, test_source) + + def test_with_stmt_with_paren(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } + statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } + compound_stmt[stmt_ty]: with_stmt + with_stmt[stmt_ty]: ( + a='with' '(' b[asdl_withitem_seq*]=','.with_item+ ')' ':' c=block { + _PyAST_With(b, (asdl_stmt_seq*) _PyPegen_singleton_seq(p, c), NULL, EXTRA) } + ) + with_item[withitem_ty]: ( + e=NAME o=['as' t=NAME { t }] { _PyAST_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) } + ) + block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a } + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) } + """ + test_source = """ + stmt = "with (\\n a as b,\\n c as d\\n): pass" + the_ast = parse.parse_string(stmt, mode=1) + self.assertStartsWith(ast_dump(the_ast), + "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " + "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" + ) + """ + self.run_test(grammar_source, test_source) + + def test_ternary_operator(self) -> None: + grammar_source = """ + start[mod_ty]: a=expr ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + expr[asdl_stmt_seq*]: a=listcomp NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, _PyAST_Expr(a, EXTRA)) } + listcomp[expr_ty]: ( + a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) } + ) + for_if_clauses[asdl_comprehension_seq*]: ( + a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })* + { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a } + ) + """ + test_source = """ + stmt = "[i for i in a if b]" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_syntax_error_for_string(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME + """ + test_source = r""" + for text in ("a b 42 b a", "\u540d \u540d 42 \u540d \u540d"): + try: + parse.parse_string(text, mode=0) + except SyntaxError as e: + tb = traceback.format_exc() + self.assertTrue('File "", line 1' in tb) + self.assertTrue(f"SyntaxError: invalid syntax" in tb) + """ + self.run_test(grammar_source, test_source) + + def test_headers_and_trailer(self) -> None: + grammar_source = """ + @header 'SOME HEADER' + @subheader 'SOME SUBHEADER' + @trailer 'SOME TRAILER' + start: expr+ NEWLINE? ENDMARKER + expr: x=NAME + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + + self.assertTrue("SOME HEADER" in parser_source) + self.assertTrue("SOME SUBHEADER" in parser_source) + self.assertTrue("SOME TRAILER" in parser_source) + + def test_error_in_rules(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME {PyTuple_New(-1)} + """ + # PyTuple_New raises SystemError if an invalid argument was passed. + test_source = """ + with self.assertRaises(SystemError): + parse.parse_string("a", mode=0) + """ + self.run_test(grammar_source, test_source) + + def test_no_soft_keywords(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: 'foo' + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + assert "expect_soft_keyword" not in parser_source + + def test_soft_keywords(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: "foo" + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + assert "expect_soft_keyword" in parser_source + + def test_soft_keywords_parse(self) -> None: + grammar_source = """ + start: "if" expr '+' expr NEWLINE + expr: NAME + """ + test_source = """ + valid_cases = ["if if + if"] + invalid_cases = ["if if"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_soft_keywords_lookahead(self) -> None: + grammar_source = """ + start: &"if" "if" expr '+' expr NEWLINE + expr: NAME + """ + test_source = """ + valid_cases = ["if if + if"] + invalid_cases = ["if if"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_forced(self) -> None: + grammar_source = """ + start: NAME &&':' | NAME + """ + test_source = """ + self.assertEqual(parse.parse_string("number :", mode=0), None) + with self.assertRaises(SyntaxError) as e: + parse.parse_string("a", mode=0) + self.assertIn("expected ':'", str(e.exception)) + """ + self.run_test(grammar_source, test_source) + + def test_forced_with_group(self) -> None: + grammar_source = """ + start: NAME &&(':' | ';') | NAME + """ + test_source = """ + self.assertEqual(parse.parse_string("number :", mode=0), None) + self.assertEqual(parse.parse_string("number ;", mode=0), None) + with self.assertRaises(SyntaxError) as e: + parse.parse_string("a", mode=0) + self.assertIn("expected (':' | ';')", e.exception.args[0]) + """ + self.run_test(grammar_source, test_source) diff --git a/Lib/test/test_peg_generator/test_first_sets.py b/Lib/test/test_peg_generator/test_first_sets.py new file mode 100644 index 00000000000..d6f8322f034 --- /dev/null +++ b/Lib/test/test_peg_generator/test_first_sets.py @@ -0,0 +1,286 @@ +import unittest + +from test import test_tools +from typing import Dict, Set + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import parse_string + from pegen.first_sets import FirstSetCalculator + from pegen.grammar import Grammar + + +class TestFirstSets(unittest.TestCase): + def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]: + grammar: Grammar = parse_string(grammar_source, GrammarParser) + return FirstSetCalculator(grammar.rules).calculate() + + def test_alternatives(self) -> None: + grammar = """ + start: expr NEWLINE? ENDMARKER + expr: A | B + A: 'a' | '-' + B: 'b' | '+' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "A": {"'a'", "'-'"}, + "B": {"'+'", "'b'"}, + "expr": {"'+'", "'a'", "'b'", "'-'"}, + "start": {"'+'", "'a'", "'b'", "'-'"}, + }, + ) + + def test_optionals(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ['a'] ['b'] 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"'c'", "'a'", "'b'"}, + "start": {"'c'", "'a'", "'b'"}, + }, + ) + + def test_repeat_with_separator(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: (term)? 'b' + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "term": {"NUMBER"}, + "sum": {"NUMBER", "'b'"}, + "start": {"'b'", "NUMBER"}, + }, + ) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: '+' ? term + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "term": {"NUMBER"}, + "sum": {"'+'", "NUMBER"}, + "start": {"'+'", "NUMBER"}, + }, + ) + + def test_optional_after(self) -> None: + grammar = """ + start: term NEWLINE + term: NUMBER ['+'] + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_optional_before(self) -> None: + grammar = """ + start: term NEWLINE + term: ['+'] NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}}, + ) + + def test_repeat_0(self) -> None: + grammar = """ + start: thing* "+" NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}}, + ) + + def test_repeat_0_with_group(self) -> None: + grammar = """ + start: ('+' '-')* term NEWLINE + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}}, + ) + + def test_repeat_1(self) -> None: + grammar = """ + start: thing+ '-' NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_repeat_1_with_group(self) -> None: + grammar = """ + start: ('+' term)+ term NEWLINE + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}} + ) + + def test_gather(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_positive_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: &'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"'a'"}, + "start": {"'a'"}, + "opt": {"'b'", "'c'", "'a'"}, + }, + ) + + def test_negative_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: !'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "opt": {"'b'", "'a'", "'c'"}, + "expr": {"'b'", "'c'"}, + "start": {"'b'", "'c'"}, + }, + ) + + def test_left_recursion(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: 'foo' + bar: 'bar' + baz: 'baz' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"NUMBER", "'-'"}, + "term": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + "foo": {"'foo'"}, + "bar": {"'bar'"}, + "baz": {"'baz'"}, + }, + ) + + def test_advance_left_recursion(self) -> None: + grammar = """ + start: NUMBER | sign start + sign: ['-'] + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}}, + ) + + def test_mutual_left_recursion(self) -> None: + grammar = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "foo": {"'D'", "'B'"}, + "bar": {"'D'"}, + "start": {"'D'", "'B'"}, + }, + ) + + def test_nasty_left_recursion(self) -> None: + # TODO: Validate this + grammar = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}}, + ) + + def test_nullable_rule(self) -> None: + grammar = """ + start: sign thing $ + sign: ['-'] + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "sign": {"", "'-'"}, + "thing": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + }, + ) + + def test_epsilon_production_in_start_rule(self) -> None: + grammar = """ + start: ['-'] $ + """ + self.assertEqual( + self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}} + ) + + def test_multiple_nullable_rules(self) -> None: + grammar = """ + start: sign thing other another $ + sign: ['-'] + thing: ['+'] + other: '*' + another: '/' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "sign": {"", "'-'"}, + "thing": {"'+'", ""}, + "start": {"'+'", "'-'", "'*'"}, + "other": {"'*'"}, + "another": {"'/'"}, + }, + ) diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py new file mode 100644 index 00000000000..857aced8ae5 --- /dev/null +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -0,0 +1,77 @@ +import unittest +from test import test_tools + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import RaiseRuleValidator, CutValidator + from pegen.testutil import parse_string + from pegen.grammar import Grammar + + +class TestPegen(unittest.TestCase): + def test_rule_with_no_collision(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '-' NAME + | NAME '+' NAME + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_rule_with_simple_collision(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '+' NAME + | NAME '+' NAME ';' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_rule_with_collision_after_some_other_rules(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '+' NAME + | NAME '*' NAME ';' + | NAME '-' NAME + | NAME '+' NAME ';' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_cut_validator(self) -> None: + grammar_source = """ + star: (OP ~ OP)* + plus: (OP ~ OP)+ + bracket: [OP ~ OP] + gather: OP.(OP ~ OP)+ + nested: [OP | NAME ~ OP] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = CutValidator(grammar) + for rule_name, rule in grammar.rules.items(): + with self.subTest(rule_name): + with self.assertRaises(ValidationError): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py new file mode 100644 index 00000000000..58ce558c548 --- /dev/null +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -0,0 +1,1132 @@ +import ast +import difflib +import io +import textwrap +import unittest + +from test import test_tools +from typing import Dict, Any +from tokenize import TokenInfo, NAME, NEWLINE, NUMBER, OP + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import parse_string, generate_parser, make_parser + from pegen.grammar import GrammarVisitor, GrammarError, Grammar + from pegen.grammar_visualizer import ASTGrammarPrinter + from pegen.parser import Parser + from pegen.parser_generator import compute_nullables, compute_left_recursives + from pegen.python_generator import PythonParserGenerator + + +class TestPegen(unittest.TestCase): + def test_parse_grammar(self) -> None: + grammar_source = """ + start: sum NEWLINE + sum: t1=term '+' t2=term { action } | term + term: NUMBER + """ + expected = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + self.assertEqual(str(grammar), textwrap.dedent(expected).strip()) + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + expected_repr = ( + "Rule('term', None, Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" + ) + self.assertEqual(repr(rules["term"]), expected_repr) + + def test_repeated_rules(self) -> None: + grammar_source = """ + start: the_rule NEWLINE + the_rule: 'b' NEWLINE + the_rule: 'a' NEWLINE + """ + with self.assertRaisesRegex(GrammarError, "Repeated rule 'the_rule'"): + parse_string(grammar_source, GrammarParser) + + def test_long_rule_str(self) -> None: + grammar_source = """ + start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one + """ + expected = """ + start: + | zero + | one + | one zero + | one one + | one zero zero + | one zero one + | one one zero + | one one one + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + self.assertEqual(str(grammar.rules["start"]), textwrap.dedent(expected).strip()) + + def test_typed_rules(self) -> None: + grammar = """ + start[int]: sum NEWLINE + sum[int]: t1=term '+' t2=term { action } | term + term[int]: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + self.assertEqual( + repr(rules["term"]), + "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))", + ) + + def test_gather(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") + self.assertStartsWith(repr(rules["start"]), + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" + ) + self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + node = parse_string("1, 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n" + ), + ], + ) + + def test_expr_grammar(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ], + ) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ('+' term)? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + ), + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' ? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1+\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n" + ), + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"), + ], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_alt_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ['+' term] + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + ), + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_repeat_0_simple(self) -> None: + grammar = """ + start: thing thing* NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + [ + TokenInfo( + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + [], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_repeat_0_complex(self) -> None: + grammar = """ + start: term ('+' term)* NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + ), + [ + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + [ + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="3", + start=(1, 8), + end=(1, 9), + line="1 + 2 + 3\n", + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + + def test_repeat_1_simple(self) -> None: + grammar = """ + start: thing thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + [ + TokenInfo( + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + ), + ], + ) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_1_complex(self) -> None: + grammar = """ + start: term ('+' term)+ NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + ), + [ + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + [ + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="3", + start=(1, 8), + end=(1, 9), + line="1 + 2 + 3\n", + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_with_sep_simple(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1, 2, 3\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n" + ), + ], + ) + + def test_left_recursive(self) -> None: + grammar_source = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: NAME+ + bar: NAME* + baz: NAME? + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + parser_class = generate_parser(grammar) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["expr"].left_recursive) + self.assertFalse(rules["term"].left_recursive) + self.assertFalse(rules["foo"].left_recursive) + self.assertFalse(rules["bar"].left_recursive) + self.assertFalse(rules["baz"].left_recursive) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + [ + [ + TokenInfo( + NUMBER, + string="1", + start=(1, 0), + end=(1, 1), + line="1 + 2 + 3\n", + ), + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + + def test_python_expr(self) -> None: + grammar = """ + start: expr NEWLINE? $ { ast.Expression(expr) } + expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | term { term } + ) + term: ( l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | factor { factor } + ) + factor: ( '(' expr ')' { expr } + | atom { atom } + ) + atom: ( n=NAME { ast.Name(id=n.string, ctx=ast.Load(), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + | n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + ) + """ + parser_class = make_parser(grammar) + node = parse_string("(1 + 2*3 + 5)/(6 - 2)\n", parser_class) + code = compile(node, "", "eval") + val = eval(code) + self.assertEqual(val, 3.0) + + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + + def test_nullable(self) -> None: + grammar_source = """ + start: sign NUMBER + sign: ['-' | '+'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + nullables = compute_nullables(rules) + self.assertNotIn(rules["start"], nullables) # Not None! + self.assertIn(rules["sign"], nullables) + + def test_advanced_left_recursive(self) -> None: + grammar_source = """ + start: NUMBER | sign start + sign: ['-'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + nullables = compute_nullables(rules) + compute_left_recursives(rules) + self.assertNotIn(rules["start"], nullables) # Not None! + self.assertIn(rules["sign"], nullables) + self.assertTrue(rules["start"].left_recursive) + self.assertFalse(rules["sign"].left_recursive) + + def test_mutually_left_recursive(self) -> None: + grammar_source = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["foo"].left_recursive) + self.assertTrue(rules["bar"].left_recursive) + genr.generate("") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class: Type[Parser] = ns["GeneratedParser"] + node = parse_string("D A C A E", parser_class) + + self.assertEqual( + node, + [ + [ + [ + [ + TokenInfo( + type=NAME, + string="D", + start=(1, 0), + end=(1, 1), + line="D A C A E", + ), + TokenInfo( + type=NAME, + string="A", + start=(1, 2), + end=(1, 3), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, + string="C", + start=(1, 4), + end=(1, 5), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, + string="A", + start=(1, 6), + end=(1, 7), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E" + ), + ], + ) + node = parse_string("B C A E", parser_class) + self.assertEqual( + node, + [ + [ + [ + TokenInfo( + type=NAME, + string="B", + start=(1, 0), + end=(1, 1), + line="B C A E", + ), + TokenInfo( + type=NAME, + string="C", + start=(1, 2), + end=(1, 3), + line="B C A E", + ), + ], + TokenInfo( + type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E" + ), + ], + TokenInfo( + type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E" + ), + ], + ) + + def test_nasty_mutually_left_recursive(self) -> None: + # This grammar does not recognize 'x - + =', much to my chagrin. + # But that's the way PEG works. + # [Breathlessly] + # The problem is that the toplevel target call + # recurses into maybe, which recognizes 'x - +', + # and then the toplevel target looks for another '+', + # which fails, so it retreats to NAME, + # which succeeds, so we end up just recognizing 'x', + # and then start fails because there's no '=' after that. + grammar_source = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + genr.generate("") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class = ns["GeneratedParser"] + with self.assertRaises(SyntaxError): + parse_string("x - + =", parser_class) + + def test_lookahead(self) -> None: + grammar = """ + start: (expr_stmt | assign_stmt) &'.' + expr_stmt: !(target '=') expr + assign_stmt: target '=' expr + expr: term ('+' term)* + target: NAME + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("foo = 12 + 12 .", parser_class) + self.maxDiff = None + self.assertEqual( + node, + [ + TokenInfo( + NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ." + ), + TokenInfo( + OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ." + ), + [ + TokenInfo( + NUMBER, + string="12", + start=(1, 6), + end=(1, 8), + line="foo = 12 + 12 .", + ), + [ + [ + TokenInfo( + OP, + string="+", + start=(1, 9), + end=(1, 10), + line="foo = 12 + 12 .", + ), + TokenInfo( + NUMBER, + string="12", + start=(1, 11), + end=(1, 13), + line="foo = 12 + 12 .", + ), + ] + ], + ], + ], + ) + + def test_named_lookahead_error(self) -> None: + grammar = """ + start: foo=!'x' NAME + """ + with self.assertRaises(SyntaxError): + make_parser(grammar) + + def test_start_leader(self) -> None: + grammar = """ + start: attr | NAME + attr: start '.' NAME + """ + # Would assert False without a special case in compute_left_recursives(). + make_parser(grammar) + + def test_opt_sequence(self) -> None: + grammar = """ + start: [NAME*] + """ + # This case was failing because of a double trailing comma at the end + # of a line in the generated source. See bpo-41044 + make_parser(grammar) + + def test_left_recursion_too_complex(self) -> None: + grammar = """ + start: foo + foo: bar '+' | baz '+' | '+' + bar: baz '-' | foo '-' | '-' + baz: foo '*' | bar '*' | '*' + """ + with self.assertRaises(ValueError) as errinfo: + make_parser(grammar) + self.assertTrue("no leader" in str(errinfo.exception.value)) + + def test_cut(self) -> None: + grammar = """ + start: '(' ~ expr ')' + expr: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("(1)", parser_class) + self.assertEqual( + node, + [ + TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"), + TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"), + TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"), + ], + ) + + def test_cut_is_local_in_rule(self) -> None: + grammar = """ + start: + | inner + | 'x' { "ok" } + inner: + | 'x' ~ 'y' + | 'x' + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_cut_is_local_in_parens(self) -> None: + # we currently don't guarantee this behavior, see gh-143054 + grammar = """ + start: + | ('x' ~ 'y' | 'x') + | 'x' { "ok" } + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_dangling_reference(self) -> None: + grammar = """ + start: foo ENDMARKER + foo: bar NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_bad_token_reference(self) -> None: + grammar = """ + start: foo + foo: NAMEE + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_missing_start(self) -> None: + grammar = """ + foo: NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_invalid_rule_name(self) -> None: + grammar = """ + start: _a b + _a: 'a' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_a'"): + parser_class = make_parser(grammar) + + def test_invalid_variable_name(self) -> None: + grammar = """ + start: a b + a: _x='a' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_x'"): + parser_class = make_parser(grammar) + + def test_invalid_variable_name_in_temporal_rule(self) -> None: + grammar = """ + start: a b + a: (_x='a' | 'b') | 'c' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_x'"): + parser_class = make_parser(grammar) + + def test_soft_keyword(self) -> None: + grammar = """ + start: + | "number" n=NUMBER { eval(n.string) } + | "string" n=STRING { n.string } + | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string } + """ + parser_class = make_parser(grammar) + self.assertEqual(parse_string("number 1", parser_class), 1) + self.assertEqual(parse_string("string 'b'", parser_class), "'b'") + self.assertEqual( + parse_string("number test 1", parser_class), "test = 1" + ) + assert ( + parse_string("string test 'b'", parser_class) == "test = 'b'" + ) + with self.assertRaises(SyntaxError): + parse_string("test 1", parser_class) + + def test_forced(self) -> None: + grammar = """ + start: NAME &&':' | NAME + """ + parser_class = make_parser(grammar) + self.assertTrue(parse_string("number :", parser_class)) + with self.assertRaises(SyntaxError) as e: + parse_string("a", parser_class) + + self.assertIn("expected ':'", str(e.exception)) + + def test_forced_with_group(self) -> None: + grammar = """ + start: NAME &&(':' | ';') | NAME + """ + parser_class = make_parser(grammar) + self.assertTrue(parse_string("number :", parser_class)) + self.assertTrue(parse_string("number ;", parser_class)) + with self.assertRaises(SyntaxError) as e: + parse_string("a", parser_class) + self.assertIn("expected (':' | ';')", e.exception.args[0]) + + def test_unreachable_explicit(self) -> None: + source = """ + start: NAME { UNREACHABLE } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit1(self) -> None: + source = """ + start: NAME | invalid_input + invalid_input: NUMBER { None } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit2(self) -> None: + source = """ + start: NAME | '(' invalid_input ')' + invalid_input: NUMBER { None } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit3(self) -> None: + source = """ + start: NAME | invalid_input { None } + invalid_input: NUMBER + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertNotIn("This is a test", out.getvalue()) + + def test_locations_in_alt_action_and_group(self) -> None: + grammar = """ + start: t=term NEWLINE? $ { ast.Expression(t) } + term: + | l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) } + | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) } + | factor + factor: + | ( + n=NAME { ast.Name(id=n.string, ctx=ast.Load(), LOCATIONS) } | + n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), LOCATIONS) } + ) + """ + parser_class = make_parser(grammar) + source = "2*3\n" + o = ast.dump(parse_string(source, parser_class).body, include_attributes=True) + p = ast.dump(ast.parse(source).body[0].value, include_attributes=True).replace( + " kind=None,", "" + ) + diff = "\n".join( + difflib.unified_diff( + o.split("\n"), p.split("\n"), "cpython", "python-pegen" + ) + ) + self.assertFalse(diff) + + +class TestGrammarVisitor: + class Visitor(GrammarVisitor): + def __init__(self) -> None: + self.n_nodes = 0 + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> None: + self.n_nodes += 1 + super().visit(node, *args, **kwargs) + + def test_parse_trivial_grammar(self) -> None: + grammar = """ + start: 'a' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + self.assertEqual(visitor.n_nodes, 6) + + def test_parse_or_grammar(self) -> None: + grammar = """ + start: rule + rule: 'a' | 'b' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/NameLeaf -> 6 + # Rule/Rhs/ -> 2 + # Alt/NamedItem/StringLeaf -> 3 + # Alt/NamedItem/StringLeaf -> 3 + + self.assertEqual(visitor.n_nodes, 14) + + def test_parse_repeat1_grammar(self) -> None: + grammar = """ + start: 'a'+ + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat1/StringLeaf -> 6 + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_repeat0_grammar(self) -> None: + grammar = """ + start: 'a'* + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat0/StringLeaf -> 6 + + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_optional_grammar(self) -> None: + grammar = """ + start: 'a' ['b'] + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/StringLeaf -> 6 + # NamedItem/Opt/Rhs/Alt/NamedItem/Stringleaf -> 6 + + self.assertEqual(visitor.n_nodes, 12) + + +class TestGrammarVisualizer(unittest.TestCase): + def test_simple_rule(self) -> None: + grammar = """ + start: 'a' 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_multiple_rules(self) -> None: + grammar = """ + start: a b + a: 'a' + b: 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──NameLeaf('a') + └──NamedItem + └──NameLeaf('b') + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'a'") + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_deep_nested_rule(self) -> None: + grammar = """ + start: 'a' ['b'['c'['d']]] + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'b'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'c'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'d'") + """ + ) + + self.assertEqual(output, expected_output) diff --git a/Lib/test/test_tools/i18n_data/ascii-escapes.pot b/Lib/test/test_tools/i18n_data/ascii-escapes.pot index 18d868b6a20..cc5a9f6ba61 100644 --- a/Lib/test/test_tools/i18n_data/ascii-escapes.pot +++ b/Lib/test/test_tools/i18n_data/ascii-escapes.pot @@ -15,30 +15,36 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" +#. Special characters that are always escaped in the POT file #: escapes.py:5 msgid "" "\"\t\n" "\r\\" msgstr "" +#. All ascii characters 0-31 #: escapes.py:8 msgid "" "\000\001\002\003\004\005\006\007\010\t\n" "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" msgstr "" +#. All ascii characters 32-126 #: escapes.py:13 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" msgstr "" +#. ascii char 127 #: escapes.py:17 msgid "\177" msgstr "" +#. some characters in the 128-255 range #: escapes.py:20 -msgid "€   ÿ" +msgid "\302\200 \302\240 ÿ" msgstr "" +#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively #: escapes.py:23 msgid "α ㄱ 𓂀" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/comments.pot b/Lib/test/test_tools/i18n_data/comments.pot new file mode 100644 index 00000000000..a1df46d453c --- /dev/null +++ b/Lib/test/test_tools/i18n_data/comments.pot @@ -0,0 +1,110 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: comments.py:4 +msgid "foo" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:7 +msgid "bar" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#: comments.py:11 +msgid "baz" +msgstr "" + +#. i18n: This is a translator comment +#. with multiple +#. lines +#: comments.py:16 +msgid "qux" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:21 +msgid "quux" +msgstr "" + +#. i18n: This is a translator comment +#. with multiple lines +#. i18n: This is another translator comment +#. with multiple lines +#: comments.py:27 +msgid "corge" +msgstr "" + +#: comments.py:31 +msgid "grault" +msgstr "" + +#. i18n: This is another translator comment +#: comments.py:36 +msgid "garply" +msgstr "" + +#: comments.py:40 +msgid "george" +msgstr "" + +#. i18n: This is another translator comment +#: comments.py:45 +msgid "waldo" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is also a translator comment +#. i18n: This is another translator comment +#: comments.py:50 +msgid "waldo2" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#. i18n: This is yet another translator comment +#. i18n: This is a translator comment +#. with multiple lines +#: comments.py:53 comments.py:56 comments.py:59 comments.py:63 +msgid "fred" +msgstr "" + +#: comments.py:65 +msgid "plugh" +msgstr "" + +#: comments.py:67 +msgid "foobar" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:71 +msgid "xyzzy" +msgstr "" + +#: comments.py:72 +msgid "thud" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#. i18n: This is yet another translator comment +#: comments.py:78 +msgid "foos" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/comments.py b/Lib/test/test_tools/i18n_data/comments.py new file mode 100644 index 00000000000..dca4dfa57b1 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/comments.py @@ -0,0 +1,78 @@ +from gettext import gettext as _ + +# Not a translator comment +_('foo') + +# i18n: This is a translator comment +_('bar') + +# i18n: This is a translator comment +# i18n: This is another translator comment +_('baz') + +# i18n: This is a translator comment +# with multiple +# lines +_('qux') + +# This comment should not be included because +# it does not start with the prefix +# i18n: This is a translator comment +_('quux') + +# i18n: This is a translator comment +# with multiple lines +# i18n: This is another translator comment +# with multiple lines +_('corge') + +# i18n: This comment should be ignored + +_('grault') + +# i18n: This comment should be ignored + +# i18n: This is another translator comment +_('garply') + +# i18n: comment should be ignored +x = 1 +_('george') + +# i18n: This comment should be ignored +x = 1 +# i18n: This is another translator comment +_('waldo') + +# i18n: This is a translator comment +x = 1 # i18n: This is also a translator comment +# i18n: This is another translator comment +_('waldo2') + +# i18n: This is a translator comment +_('fred') + +# i18n: This is another translator comment +_('fred') + +# i18n: This is yet another translator comment +_('fred') + +# i18n: This is a translator comment +# with multiple lines +_('fred') + +_('plugh') # i18n: This comment should be ignored + +_('foo' # i18n: This comment should be ignored + 'bar') # i18n: This comment should be ignored + +# i18n: This is a translator comment +_('xyzzy') +_('thud') + + +## i18n: This is a translator comment +# # i18n: This is another translator comment +### ### i18n: This is yet another translator comment +_('foos') diff --git a/Lib/test/test_tools/i18n_data/custom_keywords.pot b/Lib/test/test_tools/i18n_data/custom_keywords.pot new file mode 100644 index 00000000000..03a9cba3a20 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/custom_keywords.pot @@ -0,0 +1,51 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: custom_keywords.py:10 custom_keywords.py:11 +msgid "bar" +msgstr "" + +#: custom_keywords.py:13 +msgid "cat" +msgid_plural "cats" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:14 +msgid "dog" +msgid_plural "dogs" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:16 +msgctxt "context" +msgid "bar" +msgstr "" + +#: custom_keywords.py:18 +msgctxt "context" +msgid "cat" +msgid_plural "cats" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:34 +msgid "overridden" +msgid_plural "default" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/custom_keywords.py b/Lib/test/test_tools/i18n_data/custom_keywords.py new file mode 100644 index 00000000000..ba0ffe77180 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/custom_keywords.py @@ -0,0 +1,34 @@ +from gettext import ( + gettext as foo, + ngettext as nfoo, + pgettext as pfoo, + npgettext as npfoo, + gettext as bar, + gettext as _, +) + +foo('bar') +foo('bar', 'baz') + +nfoo('cat', 'cats', 1) +nfoo('dog', 'dogs') + +pfoo('context', 'bar') + +npfoo('context', 'cat', 'cats', 1) + +# This is an unknown keyword and should be ignored +bar('baz') + +# 'nfoo' requires at least 2 arguments +nfoo('dog') + +# 'pfoo' requires at least 2 arguments +pfoo('context') + +# 'npfoo' requires at least 3 arguments +npfoo('context') +npfoo('context', 'cat') + +# --keyword should override the default keyword +_('overridden', 'default') diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot index 5af1d41422f..387db2413a5 100644 --- a/Lib/test/test_tools/i18n_data/docstrings.pot +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -15,26 +15,40 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" -#: docstrings.py:7 +#: docstrings.py:1 +#, docstring +msgid "Module docstring" +msgstr "" + +#: docstrings.py:9 #, docstring msgid "" msgstr "" -#: docstrings.py:18 +#: docstrings.py:15 +#, docstring +msgid "docstring" +msgstr "" + +#: docstrings.py:20 #, docstring msgid "" "multiline\n" -" docstring\n" -" " +"docstring" msgstr "" -#: docstrings.py:25 +#: docstrings.py:27 #, docstring msgid "docstring1" msgstr "" -#: docstrings.py:30 +#: docstrings.py:38 +#, docstring +msgid "nested docstring" +msgstr "" + +#: docstrings.py:43 #, docstring -msgid "Hello, {}!" +msgid "nested class docstring" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py index 85d7f159d37..151a55a4b56 100644 --- a/Lib/test/test_tools/i18n_data/docstrings.py +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -1,3 +1,5 @@ +"""Module docstring""" + # Test docstring extraction from gettext import gettext as _ @@ -10,10 +12,10 @@ def test(x): # Leading empty line def test2(x): - """docstring""" # XXX This should be extracted but isn't. + """docstring""" -# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +# Multiline docstrings are cleaned with `inspect.cleandoc`. def test3(x): """multiline docstring @@ -27,15 +29,15 @@ def test4(x): def test5(x): - """Hello, {}!""".format("world!") # XXX This should not be extracted. + """Hello, {}!""".format("world!") # This should not be extracted. # Nested docstrings def test6(x): def inner(y): - """nested docstring""" # XXX This should be extracted but isn't. + """nested docstring""" class Outer: class Inner: - "nested class docstring" # XXX This should be extracted but isn't. + "nested class docstring" diff --git a/Lib/test/test_tools/i18n_data/escapes.pot b/Lib/test/test_tools/i18n_data/escapes.pot index 2c7899d59da..4dfac0f451d 100644 --- a/Lib/test/test_tools/i18n_data/escapes.pot +++ b/Lib/test/test_tools/i18n_data/escapes.pot @@ -15,30 +15,36 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" +#. Special characters that are always escaped in the POT file #: escapes.py:5 msgid "" "\"\t\n" "\r\\" msgstr "" +#. All ascii characters 0-31 #: escapes.py:8 msgid "" "\000\001\002\003\004\005\006\007\010\t\n" "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" msgstr "" +#. All ascii characters 32-126 #: escapes.py:13 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" msgstr "" +#. ascii char 127 #: escapes.py:17 msgid "\177" msgstr "" +#. some characters in the 128-255 range #: escapes.py:20 msgid "\302\200 \302\240 \303\277" msgstr "" +#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively #: escapes.py:23 msgid "\316\261 \343\204\261 \360\223\202\200" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot index ddfbd18349e..e8167acfc07 100644 --- a/Lib/test/test_tools/i18n_data/messages.pot +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -15,53 +15,85 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" -#: messages.py:5 +#: messages.py:16 msgid "" msgstr "" -#: messages.py:8 messages.py:9 +#: messages.py:19 messages.py:20 messages.py:21 msgid "parentheses" msgstr "" -#: messages.py:12 +#: messages.py:24 msgid "Hello, world!" msgstr "" -#: messages.py:15 +#: messages.py:27 msgid "" "Hello,\n" " multiline!\n" msgstr "" -#: messages.py:29 +#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94 +#: messages.py:99 messages.py:100 messages.py:101 +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:47 +msgid "something" +msgstr "" + +#: messages.py:50 msgid "Hello, {}!" msgstr "" -#: messages.py:33 +#: messages.py:54 msgid "1" msgstr "" -#: messages.py:33 +#: messages.py:54 msgid "2" msgstr "" -#: messages.py:34 messages.py:35 +#: messages.py:55 messages.py:56 msgid "A" msgstr "" -#: messages.py:34 messages.py:35 +#: messages.py:55 messages.py:56 msgid "B" msgstr "" -#: messages.py:36 +#: messages.py:57 msgid "set" msgstr "" -#: messages.py:42 +#: messages.py:62 messages.py:63 msgid "nested string" msgstr "" -#: messages.py:47 +#: messages.py:68 msgid "baz" msgstr "" +#: messages.py:71 messages.py:75 +msgid "default value" +msgstr "" + +#: messages.py:91 messages.py:92 messages.py:95 messages.py:96 +msgctxt "context" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:102 +msgid "domain foo" +msgstr "" + +#: messages.py:118 messages.py:119 +msgid "world" +msgid_plural "worlds" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py index f220294b8d5..9457bcb8611 100644 --- a/Lib/test/test_tools/i18n_data/messages.py +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -1,5 +1,16 @@ # Test message extraction -from gettext import gettext as _ +from gettext import ( + gettext, + ngettext, + pgettext, + npgettext, + dgettext, + dngettext, + dpgettext, + dnpgettext +) + +_ = gettext # Empty string _("") @@ -7,6 +18,7 @@ # Extra parentheses (_("parentheses")) ((_("parentheses"))) +_(("parentheses")) # Multiline strings _("Hello, " @@ -21,13 +33,22 @@ _(None) _(1) _(False) -_(x="kwargs are not allowed") +_(["invalid"]) +_({"invalid"}) +_("string"[3]) +_("string"[:3]) +_({"string": "foo"}) + +# pygettext does not allow keyword arguments, but both xgettext and pybabel do +_(x="kwargs are not allowed!") + +# Unusual, but valid arguments _("foo", "bar") _("something", x="something else") # .format() _("Hello, {}!").format("world") # valid -_("Hello, {}!".format("world")) # invalid +_("Hello, {}!".format("world")) # invalid, but xgettext extracts the first string # Nested structures _("1"), _("2") @@ -38,7 +59,7 @@ # Nested functions and classes def test(): - _("nested string") # XXX This should be extracted but isn't. + _("nested string") [_("nested string")] @@ -47,11 +68,11 @@ def bar(self): return _("baz") -def bar(x=_('default value')): # XXX This should be extracted but isn't. +def bar(x=_('default value')): pass -def baz(x=[_('default value')]): # XXX This should be extracted but isn't. +def baz(x=[_('default value')]): pass @@ -62,3 +83,37 @@ def _(x): def _(x="don't extract me"): pass + + +# Other gettext functions +gettext("foo") +ngettext("foo", "foos", 1) +pgettext("context", "foo") +npgettext("context", "foo", "foos", 1) +dgettext("domain", "foo") +dngettext("domain", "foo", "foos", 1) +dpgettext("domain", "context", "foo") +dnpgettext("domain", "context", "foo", "foos", 1) + +# Complex arguments +ngettext("foo", "foos", 42 + (10 - 20)) +ngettext("foo", "foos", *args) +ngettext("foo", "foos", **kwargs) +dgettext(["some", {"complex"}, ("argument",)], "domain foo") + +# Invalid calls which are not extracted +gettext() +ngettext('foo') +pgettext('context') +npgettext('context', 'foo') +dgettext('domain') +dngettext('domain', 'foo') +dpgettext('domain', 'context') +dnpgettext('domain', 'context', 'foo') +dgettext(*args, 'foo') +dpgettext(*args, 'context', 'foo') +dnpgettext(*args, 'context', 'foo', 'foos') + +# f-strings +f"Hello, {_('world')}!" +f"Hello, {ngettext('world', 'worlds', 3)}!" diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.pot b/Lib/test/test_tools/i18n_data/multiple_keywords.pot new file mode 100644 index 00000000000..954cb8e9948 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.pot @@ -0,0 +1,38 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: multiple_keywords.py:3 +msgid "bar" +msgstr "" + +#: multiple_keywords.py:5 +msgctxt "baz" +msgid "qux" +msgstr "" + +#: multiple_keywords.py:9 +msgctxt "corge" +msgid "grault" +msgstr "" + +#: multiple_keywords.py:11 +msgctxt "xyzzy" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.py b/Lib/test/test_tools/i18n_data/multiple_keywords.py new file mode 100644 index 00000000000..7bde349505b --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.py @@ -0,0 +1,11 @@ +from gettext import gettext as foo + +foo('bar') + +foo('baz', 'qux') + +# The 't' specifier is not supported, so the following +# call is extracted as pgettext instead of ngettext. +foo('corge', 'grault', 1) + +foo('xyzzy', 'foo', 'foos', 1) diff --git a/Lib/test/test_tools/test_compute_changes.py b/Lib/test/test_tools/test_compute_changes.py new file mode 100644 index 00000000000..b20ff975fc2 --- /dev/null +++ b/Lib/test/test_tools/test_compute_changes.py @@ -0,0 +1,144 @@ +"""Tests to cover the Tools/build/compute-changes.py script.""" + +import importlib +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from test.test_tools import skip_if_missing, imports_under_tool + +skip_if_missing("build") + +with patch.dict(os.environ, {"GITHUB_DEFAULT_BRANCH": "main"}): + with imports_under_tool("build"): + compute_changes = importlib.import_module("compute-changes") + +process_changed_files = compute_changes.process_changed_files +Outputs = compute_changes.Outputs +ANDROID_DIRS = compute_changes.ANDROID_DIRS +IOS_DIRS = compute_changes.IOS_DIRS +MACOS_DIRS = compute_changes.MACOS_DIRS +WASI_DIRS = compute_changes.WASI_DIRS +RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE +UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES +LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS + + +class TestProcessChangedFiles(unittest.TestCase): + + def test_windows(self): + f = {Path(".github/workflows/reusable-windows.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_windows_tests) + + def test_docs(self): + for f in ( + ".github/workflows/reusable-docs.yml", + "Doc/library/datetime.rst", + "Doc/Makefile", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_docs) + self.assertFalse(result.run_tests) + + def test_ci_fuzz_stdlib(self): + for p in LIBRARY_FUZZER_PATHS: + with self.subTest(p=p): + if p.is_dir(): + f = p / "file" + elif p.is_file(): + f = p + else: + continue + result = process_changed_files({f}) + self.assertTrue(result.run_ci_fuzz_stdlib) + + def test_android(self): + for d in ANDROID_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertFalse(result.run_windows_tests) + + def test_ios(self): + for d in IOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_ios) + self.assertFalse(result.run_windows_tests) + + def test_macos(self): + f = {Path(".github/workflows/reusable-macos.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + + for d in MACOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + self.assertFalse(result.run_windows_tests) + + def test_wasi(self): + f = {Path(".github/workflows/reusable-wasi.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + for d in WASI_DIRS: + with self.subTest(d=d): + result = process_changed_files({d / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + self.assertFalse(result.run_windows_tests) + + def test_unix(self): + for f in UNIX_BUILD_SYSTEM_FILE_NAMES: + with self.subTest(f=f): + result = process_changed_files({f}) + self.assertTrue(result.run_tests) + self.assertFalse(result.run_windows_tests) + + def test_msi(self): + for f in ( + ".github/workflows/reusable-windows-msi.yml", + "Tools/msi/build.bat", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_windows_msi) + + def test_all_run(self): + for f in ( + ".github/workflows/some-new-workflow.yml", + ".github/workflows/build.yml", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertTrue(result.run_ios) + self.assertTrue(result.run_macos) + self.assertTrue(result.run_ubuntu) + self.assertTrue(result.run_wasi) + + def test_all_ignored(self): + for f in RUN_TESTS_IGNORE: + with self.subTest(f=f): + self.assertEqual(process_changed_files({Path(f)}), Outputs()) + + def test_wasi_and_android(self): + f = {Path(".github/workflows/reusable-wasi.yml"), Path("Android/file")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index ffa1b1178ed..d1831d68f02 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -8,7 +8,7 @@ from pathlib import Path from test.support.script_helper import assert_python_ok -from test.test_tools import skip_if_missing, toolsdir +from test.test_tools import imports_under_tool, skip_if_missing, toolsdir from test.support.os_helper import temp_cwd, temp_dir @@ -17,6 +17,11 @@ DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' +with imports_under_tool("i18n"): + from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS, + unparse_spec) + + def normalize_POT_file(pot): """Normalize the POT creation timestamp, charset and file locations to make the POT file easier to compare. @@ -87,7 +92,8 @@ def assert_POT_equal(self, expected, actual): self.maxDiff = None self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) - def extract_from_str(self, module_content, *, args=(), strict=True): + def extract_from_str(self, module_content, *, args=(), strict=True, + with_stderr=False, raw=False): """Return all msgids extracted from module_content.""" filename = 'test.py' with temp_cwd(None): @@ -98,12 +104,19 @@ def extract_from_str(self, module_content, *, args=(), strict=True): self.assertEqual(res.err, b'') with open('messages.pot', encoding='utf-8') as fp: data = fp.read() - return self.get_msgids(data) + if not raw: + data = self.get_msgids(data) + if not with_stderr: + return data + return data, res.err def extract_docstrings_from_str(self, module_content): """Return all docstrings extracted from module_content.""" return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + def get_stderr(self, module_content): + return self.extract_from_str(module_content, strict=False, with_stderr=True)[1] + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry @@ -149,6 +162,14 @@ def test_POT_Creation_Date(self): # This will raise if the date format does not exactly match. datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + def test_output_option(self): + for opt in ('-o', '--output='): + with temp_cwd(): + assert_python_ok(self.script, f'{opt}test') + self.assertTrue(os.path.exists('test')) + res = assert_python_ok(self.script, f'{opt}-') + self.assertIn(b'Project-Id-Version: PACKAGE VERSION', res.out) + def test_funcdocstring(self): for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): with self.subTest(doc): @@ -332,14 +353,14 @@ def test_calls_in_fstring_with_multiple_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', 'bar')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) def test_calls_in_fstring_with_keyword_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', bar='baz')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) self.assertNotIn('baz', msgids) @@ -400,17 +421,195 @@ def test_files_list(self): self.assertIn(f'msgid "{text2}"', data) self.assertNotIn(text3, data) + def test_help_text(self): + """Test that the help text is displayed.""" + res = assert_python_ok(self.script, '--help') + self.assertEqual(res.out, b'') + self.assertIn(b'pygettext -- Python equivalent of xgettext(1)', res.err) + + def test_error_messages(self): + """Test that pygettext outputs error messages to stderr.""" + stderr = self.get_stderr(dedent('''\ + _(1+2) + ngettext('foo') + dgettext(*args, 'foo') + ''')) + + # Normalize line endings on Windows + stderr = stderr.decode('utf-8').replace('\r', '') + + self.assertEqual( + stderr, + "*** test.py:1: Expected a string constant for argument 1, got 1 + 2\n" + "*** test.py:2: Expected at least 2 positional argument(s) in gettext call, got 1\n" + "*** test.py:3: Variable positional arguments are not allowed in gettext calls\n" + ) + + def test_extract_all_comments(self): + """ + Test that the --add-comments option without an + explicit tag extracts all translator comments. + """ + for arg in ('--add-comments', '-c'): + with self.subTest(arg=arg): + data = self.extract_from_str(dedent('''\ + # Translator comment + _("foo") + '''), args=(arg,), raw=True) + self.assertIn('#. Translator comment', data) + + def test_comments_with_multiple_tags(self): + """ + Test that multiple --add-comments tags can be specified. + """ + for arg in ('--add-comments={}', '-c{}'): + with self.subTest(arg=arg): + args = (arg.format('foo:'), arg.format('bar:')) + data = self.extract_from_str(dedent('''\ + # foo: comment + _("foo") + + # bar: comment + _("bar") + + # baz: comment + _("baz") + '''), args=args, raw=True) + self.assertIn('#. foo: comment', data) + self.assertIn('#. bar: comment', data) + self.assertNotIn('#. baz: comment', data) + + def test_comments_not_extracted_without_tags(self): + """ + Test that translator comments are not extracted without + specifying --add-comments. + """ + data = self.extract_from_str(dedent('''\ + # Translator comment + _("foo") + '''), raw=True) + self.assertNotIn('#.', data) + + def test_parse_keyword_spec(self): + valid = ( + ('foo', ('foo', {'msgid': 0})), + ('foo:1', ('foo', {'msgid': 0})), + ('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ) + for spec, expected in valid: + with self.subTest(spec=spec): + self.assertEqual(parse_spec(spec), expected) + # test unparse-parse round-trip + self.assertEqual(parse_spec(unparse_spec(*expected)), expected) + + invalid = ( + ('foo:', "Invalid keyword spec 'foo:': missing argument positions"), + ('foo:bar', "Invalid keyword spec 'foo:bar': position is not an integer"), + ('foo:0', "Invalid keyword spec 'foo:0': argument positions must be strictly positive"), + ('foo:-2', "Invalid keyword spec 'foo:-2': argument positions must be strictly positive"), + ('foo:1,1', "Invalid keyword spec 'foo:1,1': duplicate positions"), + ('foo:1,2,1', "Invalid keyword spec 'foo:1,2,1': duplicate positions"), + ('foo:1c,2,1c', "Invalid keyword spec 'foo:1c,2,1c': duplicate positions"), + ('foo:1c,2,3c', "Invalid keyword spec 'foo:1c,2,3c': msgctxt can only appear once"), + ('foo:1,2,3', "Invalid keyword spec 'foo:1,2,3': too many positions"), + ('foo:1c', "Invalid keyword spec 'foo:1c': msgctxt cannot appear without msgid"), + ) + for spec, message in invalid: + with self.subTest(spec=spec): + with self.assertRaises(ValueError) as cm: + parse_spec(spec) + self.assertEqual(str(cm.exception), message) + + def test_process_keywords(self): + default_keywords = {name: [spec] for name, spec + in DEFAULTKEYWORDS.items()} + inputs = ( + (['foo'], True), + (['_:1,2'], True), + (['foo', 'foo:1,2'], True), + (['foo'], False), + (['_:1,2', '_:1c,2,3', 'pgettext'], False), + # Duplicate entries + (['foo', 'foo'], True), + (['_'], False) + ) + expected = ( + {'foo': [{'msgid': 0}]}, + {'_': [{'msgid': 0, 'msgid_plural': 1}]}, + {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]}, + default_keywords | {'foo': [{'msgid': 0}]}, + default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1}, + {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}, + {'msgid': 0}], + 'pgettext': [{'msgid': 0}, + {'msgctxt': 0, 'msgid': 1}]}, + {'foo': [{'msgid': 0}]}, + default_keywords, + ) + for (keywords, no_default_keywords), expected in zip(inputs, expected): + with self.subTest(keywords=keywords, + no_default_keywords=no_default_keywords): + processed = process_keywords( + keywords, + no_default_keywords=no_default_keywords) + self.assertEqual(processed, expected) + + def test_multiple_keywords_same_funcname_errors(self): + # If at least one keyword spec for a given funcname matches, + # no error should be printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _("foo", 42) + _(42, "bar") + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True) + self.assertIn('foo', msgids) + self.assertIn('bar', msgids) + self.assertEqual(stderr, b'') + + # If no keyword spec for a given funcname matches, + # all errors are printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _(x, 42) + _(42, y) + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True, + strict=False) + self.assertEqual(msgids, ['']) + # Normalize line endings on Windows + stderr = stderr.decode('utf-8').replace('\r', '') + self.assertEqual( + stderr, + '*** test.py:1: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got x\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got 42\n' + '*** test.py:2: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got 42\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got y\n') + def extract_from_snapshots(): snapshots = { - 'messages.py': ('--docstrings',), + 'messages.py': (), 'fileloc.py': ('--docstrings',), 'docstrings.py': ('--docstrings',), + 'comments.py': ('--add-comments=i18n:',), + 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', + '--keyword=pfoo:1c,2', + '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'), + 'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2', + '--keyword=foo:1,2', + # repeat a keyword to make sure it is extracted only once + '--keyword=foo', '--keyword=foo'), # == Test character escaping # Escape ascii and unicode: - 'escapes.py': ('--escape',), + 'escapes.py': ('--escape', '--add-comments='), # Escape only ascii and let unicode pass through: - ('escapes.py', 'ascii-escapes.pot'): (), + ('escapes.py', 'ascii-escapes.pot'): ('--add-comments=',), } for filename, args in snapshots.items(): diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 4c7588d4d93..31a51606739 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -48,15 +48,18 @@ def test_makefile_test_folders(self): if dirname == '__pycache__' or dirname.startswith('.'): dirs.clear() # do not process subfolders continue - # Skip empty dirs: + + # Skip empty dirs (ignoring hidden files and __pycache__): + files = [ + filename for filename in files + if not filename.startswith('.') + ] + dirs = [ + dirname for dirname in dirs + if not dirname.startswith('.') and dirname != "__pycache__" + ] if not dirs and not files: continue - # Skip dirs with hidden-only files: - if files and all( - filename.startswith('.') or filename == '__pycache__' - for filename in files - ): - continue relpath = os.path.relpath(dirpath, support.STDLIB_DIR) with self.subTest(relpath=relpath): diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py index 8cd31680f76..7be606bbff6 100644 --- a/Lib/test/test_tools/test_msgfmt.py +++ b/Lib/test/test_tools/test_msgfmt.py @@ -1,6 +1,7 @@ """Tests for the Tools/i18n/msgfmt.py tool.""" import json +import struct import sys import unittest from gettext import GNUTranslations @@ -8,18 +9,21 @@ from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok -from test.test_tools import skip_if_missing, toolsdir +from test.test_tools import imports_under_tool, skip_if_missing, toolsdir skip_if_missing('i18n') data_dir = (Path(__file__).parent / 'msgfmt_data').resolve() script_dir = Path(toolsdir) / 'i18n' -msgfmt = script_dir / 'msgfmt.py' +msgfmt_py = script_dir / 'msgfmt.py' + +with imports_under_tool("i18n"): + import msgfmt def compile_messages(po_file, mo_file): - assert_python_ok(msgfmt, '-o', mo_file, po_file) + assert_python_ok(msgfmt_py, '-o', mo_file, po_file) class CompilationTest(unittest.TestCase): @@ -40,6 +44,31 @@ def test_compilation(self): self.assertDictEqual(actual._catalog, expected._catalog) + def test_binary_header(self): + with temp_cwd(): + tmp_mo_file = 'messages.mo' + compile_messages(data_dir / "general.po", tmp_mo_file) + with open(tmp_mo_file, 'rb') as f: + mo_data = f.read() + + ( + magic, + version, + num_strings, + orig_table_offset, + trans_table_offset, + hash_table_size, + hash_table_offset, + ) = struct.unpack("=7I", mo_data[:28]) + + self.assertEqual(magic, 0x950412de) + self.assertEqual(version, 0) + self.assertEqual(num_strings, 9) + self.assertEqual(orig_table_offset, 28) + self.assertEqual(trans_table_offset, 100) + self.assertEqual(hash_table_size, 0) + self.assertEqual(hash_table_offset, 0) + def test_translations(self): with open(data_dir / 'general.mo', 'rb') as f: t = GNUTranslations(f) @@ -62,6 +91,14 @@ def test_translations(self): '%d emails sent.', 2), '%d emails sent.') + def test_po_with_bom(self): + with temp_cwd(): + Path('bom.po').write_bytes(b'\xef\xbb\xbfmsgid "Python"\nmsgstr "Pioton"\n') + + res = assert_python_failure(msgfmt_py, 'bom.po') + err = res.err.decode('utf-8') + self.assertIn('The file bom.po starts with a UTF-8 BOM', err) + def test_invalid_msgid_plural(self): with temp_cwd(): Path('invalid.po').write_text('''\ @@ -69,7 +106,7 @@ def test_invalid_msgid_plural(self): msgstr[0] "singular" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('msgid_plural not preceded by msgid', err) @@ -80,7 +117,7 @@ def test_plural_without_msgid_plural(self): msgstr[0] "bar" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('plural without msgid_plural', err) @@ -92,7 +129,7 @@ def test_indexed_msgstr_without_msgid_plural(self): msgstr "bar" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('indexed msgstr required for plural', err) @@ -102,38 +139,136 @@ def test_generic_syntax_error(self): "foo" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('Syntax error', err) + +class POParserTest(unittest.TestCase): + @classmethod + def tearDownClass(cls): + # msgfmt uses a global variable to store messages, + # clear it after the tests. + msgfmt.MESSAGES.clear() + + def test_strings(self): + # Test that the PO parser correctly handles and unescape + # strings in the PO file. + # The PO file format allows for a variety of escape sequences, + # octal and hex escapes. + valid_strings = ( + # empty strings + ('""', ''), + ('"" "" ""', ''), + # allowed escape sequences + (r'"\\"', '\\'), + (r'"\""', '"'), + (r'"\t"', '\t'), + (r'"\n"', '\n'), + (r'"\r"', '\r'), + (r'"\f"', '\f'), + (r'"\a"', '\a'), + (r'"\b"', '\b'), + (r'"\v"', '\v'), + # non-empty strings + ('"foo"', 'foo'), + ('"foo" "bar"', 'foobar'), + ('"foo""bar"', 'foobar'), + ('"" "foo" ""', 'foo'), + # newlines and tabs + (r'"foo\nbar"', 'foo\nbar'), + (r'"foo\n" "bar"', 'foo\nbar'), + (r'"foo\tbar"', 'foo\tbar'), + (r'"foo\t" "bar"', 'foo\tbar'), + # escaped quotes + (r'"foo\"bar"', 'foo"bar'), + (r'"foo\"" "bar"', 'foo"bar'), + (r'"foo\\" "bar"', 'foo\\bar'), + # octal escapes + (r'"\120\171\164\150\157\156"', 'Python'), + (r'"\120\171\164" "\150\157\156"', 'Python'), + (r'"\"\120\171\164" "\150\157\156\""', '"Python"'), + # hex escapes + (r'"\x50\x79\x74\x68\x6f\x6e"', 'Python'), + (r'"\x50\x79\x74" "\x68\x6f\x6e"', 'Python'), + (r'"\"\x50\x79\x74" "\x68\x6f\x6e\""', '"Python"'), + ) + + with temp_cwd(): + for po_string, expected in valid_strings: + with self.subTest(po_string=po_string): + # Construct a PO file with a single entry, + # compile it, read it into a catalog and + # check the result. + po = f'msgid {po_string}\nmsgstr "translation"' + Path('messages.po').write_text(po) + # Reset the global MESSAGES dictionary + msgfmt.MESSAGES.clear() + msgfmt.make('messages.po', 'messages.mo') + + with open('messages.mo', 'rb') as f: + actual = GNUTranslations(f) + + self.assertDictEqual(actual._catalog, {expected: 'translation'}) + + invalid_strings = ( + # "''", # invalid but currently accepted + '"', + '"""', + '"" "', + 'foo', + '"" "foo', + '"foo" foo', + '42', + '"" 42 ""', + # disallowed escape sequences + # r'"\'"', # invalid but currently accepted + # r'"\e"', # invalid but currently accepted + # r'"\8"', # invalid but currently accepted + # r'"\9"', # invalid but currently accepted + r'"\x"', + r'"\u1234"', + r'"\N{ROMAN NUMERAL NINE}"' + ) + with temp_cwd(): + for invalid_string in invalid_strings: + with self.subTest(string=invalid_string): + po = f'msgid {invalid_string}\nmsgstr "translation"' + Path('messages.po').write_text(po) + # Reset the global MESSAGES dictionary + msgfmt.MESSAGES.clear() + with self.assertRaises(Exception): + msgfmt.make('messages.po', 'messages.mo') + + class CLITest(unittest.TestCase): def test_help(self): for option in ('--help', '-h'): - res = assert_python_ok(msgfmt, option) + res = assert_python_ok(msgfmt_py, option) err = res.err.decode('utf-8') self.assertIn('Generate binary message catalog from textual translation description.', err) def test_version(self): for option in ('--version', '-V'): - res = assert_python_ok(msgfmt, option) + res = assert_python_ok(msgfmt_py, option) out = res.out.decode('utf-8').strip() self.assertEqual('msgfmt.py 1.2', out) def test_invalid_option(self): - res = assert_python_failure(msgfmt, '--invalid-option') + res = assert_python_failure(msgfmt_py, '--invalid-option') err = res.err.decode('utf-8') self.assertIn('Generate binary message catalog from textual translation description.', err) self.assertIn('option --invalid-option not recognized', err) def test_no_input_file(self): - res = assert_python_ok(msgfmt) + res = assert_python_ok(msgfmt_py) err = res.err.decode('utf-8').replace('\r\n', '\n') self.assertIn('No input file given\n' "Try `msgfmt --help' for more information.", err) def test_nonexistent_file(self): - assert_python_failure(msgfmt, 'nonexistent.po') + assert_python_failure(msgfmt_py, 'nonexistent.po') def update_catalog_snapshots(): diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py new file mode 100644 index 00000000000..7469a1047f8 --- /dev/null +++ b/Lib/test/test_type_cache.py @@ -0,0 +1,265 @@ +""" Tests for the internal type cache in CPython. """ +import dis +import unittest +import warnings +from test import support +from test.support import import_helper, requires_specialization, requires_specialization_ft +try: + from sys import _clear_type_cache +except ImportError: + _clear_type_cache = None + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") +type_get_version = _testcapi.type_get_version +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe +type_assign_version = _testcapi.type_assign_version +type_modified = _testcapi.type_modified + +def clear_type_cache(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _clear_type_cache() + +@support.cpython_only +@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") +class TypeCacheTests(unittest.TestCase): + def test_tp_version_tag_unique(self): + """tp_version_tag should be unique assuming no overflow, even after + clearing type cache. + """ + # Check if global version tag has already overflowed. + Y = type('Y', (), {}) + Y.x = 1 + Y.x # Force a _PyType_Lookup, populating version tag + y_ver = type_get_version(Y) + # Overflow, or not enough left to conduct the test. + if y_ver == 0 or y_ver > 0xFFFFF000: + self.skipTest("Out of type version tags") + # Note: try to avoid any method lookups within this loop, + # It will affect global version tag. + all_version_tags = [] + append_result = all_version_tags.append + assertNotEqual = self.assertNotEqual + for _ in range(30): + clear_type_cache() + X = type('Y', (), {}) + X.x = 1 + X.x + tp_version_tag_after = type_get_version(X) + assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") + append_result(tp_version_tag_after) + self.assertEqual(len(set(all_version_tags)), 30, + msg=f"{all_version_tags} contains non-unique versions") + + def test_type_assign_version(self): + class C: + x = 5 + + self.assertEqual(type_assign_version(C), 1) + c_ver = type_get_version(C) + + C.x = 6 + self.assertEqual(type_get_version(C), 0) + self.assertEqual(type_assign_version(C), 1) + self.assertNotEqual(type_get_version(C), 0) + self.assertNotEqual(type_get_version(C), c_ver) + + def test_type_assign_specific_version(self): + """meta-test for type_assign_specific_version_unsafe""" + class C: + pass + + type_assign_version(C) + orig_version = type_get_version(C) + if orig_version == 0: + self.skipTest("Could not assign a valid type version") + + type_modified(C) + type_assign_specific_version_unsafe(C, orig_version + 5) + type_assign_version(C) # this should do nothing + + new_version = type_get_version(C) + self.assertEqual(new_version, orig_version + 5) + + clear_type_cache() + + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + + def test_119462(self): + + class Holder: + value = None + + @classmethod + def set_value(cls): + cls.value = object() + + class HolderSub(Holder): + pass + + for _ in range(1050): + Holder.set_value() + HolderSub.value + +@support.cpython_only +class TypeCacheWithSpecializationTests(unittest.TestCase): + def tearDown(self): + clear_type_cache() + + def _assign_valid_version_or_skip(self, type_): + type_modified(type_) + type_assign_version(type_) + if type_get_version(type_) == 0: + self.skipTest("Could not assign valid type version") + + def _no_more_versions(self, user_type): + type_modified(user_type) + for _ in range(1001): + type_assign_specific_version_unsafe(user_type, 1000_000_000) + type_assign_specific_version_unsafe(user_type, 0) + self.assertEqual(type_get_version(user_type), 0) + + def _all_opnames(self, func): + return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) + + def _check_specialization(self, func, arg, opname, *, should_specialize): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + func(arg) + + if should_specialize: + self.assertNotIn(opname, self._all_opnames(func)) + else: + self.assertIn(opname, self._all_opnames(func)) + + @requires_specialization + def test_class_load_attr_specialization_user_type(self): + class A: + def foo(self): + pass + + self._assign_valid_version_or_skip(A) + + def load_foo_1(type_): + type_.foo + + self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) + del load_foo_1 + + self._no_more_versions(A) + + def load_foo_2(type_): + return type_.foo + + self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + + @requires_specialization + def test_class_load_attr_specialization_static_type(self): + self.assertNotEqual(type_get_version(str), 0) + self.assertNotEqual(type_get_version(bytes), 0) + + def get_capitalize_1(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) + self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + + @requires_specialization + def test_property_load_attr_specialization_user_type(self): + class G: + @property + def x(self): + return 9 + + self._assign_valid_version_or_skip(G) + + def load_x_1(instance): + instance.x + + self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) + del load_x_1 + + self._no_more_versions(G) + + def load_x_2(instance): + instance.x + + self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + + @requires_specialization + def test_store_attr_specialization_user_type(self): + class B: + __slots__ = ("bar",) + + self._assign_valid_version_or_skip(B) + + def store_bar_1(type_): + type_.bar = 10 + + self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) + del store_bar_1 + + self._no_more_versions(B) + + def store_bar_2(type_): + type_.bar = 10 + + self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + + @requires_specialization_ft + def test_class_call_specialization_user_type(self): + class F: + def __init__(self): + pass + + self._assign_valid_version_or_skip(F) + + def call_class_1(type_): + type_() + + self._check_specialization(call_class_1, F, "CALL", should_specialize=True) + del call_class_1 + + self._no_more_versions(F) + + def call_class_2(type_): + type_() + + self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + + @requires_specialization + def test_to_bool_specialization_user_type(self): + class H: + pass + + self._assign_valid_version_or_skip(H) + + def to_bool_1(instance): + not instance + + self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) + del to_bool_1 + + self._no_more_versions(H) + + def to_bool_2(instance): + not instance + + self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py new file mode 100644 index 00000000000..b52e78bc4fb --- /dev/null +++ b/Lib/test/test_xxlimited.py @@ -0,0 +1,90 @@ +import unittest +from test.support import import_helper +import types + +xxlimited = import_helper.import_module('xxlimited') +xxlimited_35 = import_helper.import_module('xxlimited_35') + + +class CommonTests: + module: types.ModuleType + + def test_xxo_new(self): + xxo = self.module.Xxo() + + def test_xxo_attributes(self): + xxo = self.module.Xxo() + with self.assertRaises(AttributeError): + xxo.foo + with self.assertRaises(AttributeError): + del xxo.foo + + xxo.foo = 1234 + self.assertEqual(xxo.foo, 1234) + + del xxo.foo + with self.assertRaises(AttributeError): + xxo.foo + + def test_foo(self): + # the foo function adds 2 numbers + self.assertEqual(self.module.foo(1, 2), 3) + + def test_str(self): + self.assertIsSubclass(self.module.Str, str) + self.assertIsNot(self.module.Str, str) + + custom_string = self.module.Str("abcd") + self.assertEqual(custom_string, "abcd") + self.assertEqual(custom_string.upper(), "ABCD") + + def test_new(self): + xxo = self.module.new() + self.assertEqual(xxo.demo("abc"), "abc") + + +class TestXXLimited(CommonTests, unittest.TestCase): + module = xxlimited + + def test_xxo_demo(self): + xxo = self.module.Xxo() + other = self.module.Xxo() + self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(xxo), xxo) + self.assertEqual(xxo.demo(other), other) + self.assertEqual(xxo.demo(0), None) + + def test_error(self): + with self.assertRaises(self.module.Error): + raise self.module.Error + + def test_buffer(self): + xxo = self.module.Xxo() + self.assertEqual(xxo.x_exports, 0) + b1 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 1) + b2 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 2) + b1[0] = 1 + self.assertEqual(b1[0], 1) + self.assertEqual(b2[0], 1) + + +class TestXXLimited35(CommonTests, unittest.TestCase): + module = xxlimited_35 + + def test_xxo_demo(self): + xxo = self.module.Xxo() + other = self.module.Xxo() + self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + + def test_roj(self): + # the roj function always fails + with self.assertRaises(SystemError): + self.module.roj(0) + + def test_null(self): + null1 = self.module.Null() + null2 = self.module.Null() + self.assertNotEqual(null1, null2) diff --git a/Lib/test/test_xxtestfuzz.py b/Lib/test/test_xxtestfuzz.py new file mode 100644 index 00000000000..3304c6e703a --- /dev/null +++ b/Lib/test/test_xxtestfuzz.py @@ -0,0 +1,25 @@ +import faulthandler +from test.support import import_helper +import unittest + +_xxtestfuzz = import_helper.import_module('_xxtestfuzz') + + +class TestFuzzer(unittest.TestCase): + """To keep our https://github.com/google/oss-fuzz API working.""" + + def test_sample_input_smoke_test(self): + """This is only a regression test: Check that it doesn't crash.""" + _xxtestfuzz.run(b"") + _xxtestfuzz.run(b"\0") + _xxtestfuzz.run(b"{") + _xxtestfuzz.run(b" ") + _xxtestfuzz.run(b"x") + _xxtestfuzz.run(b"1") + _xxtestfuzz.run(b"AAAAAAA") + _xxtestfuzz.run(b"AAAAAA\0") + + +if __name__ == "__main__": + faulthandler.enable() + unittest.main()