Skip to content

Commit fa8600c

Browse files
lysnikolaoulysnikolaou
authored
gh-159799: Require pathlib.[Pure]Path for file paths in the PEG generator (#159800)
The pegen entry points now take `pathlib.Path` objects instead of strings, and the command-line tools (`python -m pegen`, `keywordgen`, `first_sets`, `grammar_visualizer`) parse their path arguments into `Path` objects. The generators' `generate` and `prepare` take a `pathlib.PurePath` since they only use the file's name (for the "generated by" comment in the output) and callers pass names that are not files, such as `"<string>"`.
1 parent 01a683a commit fa8600c

11 files changed

Lines changed: 99 additions & 87 deletions

File tree

‎Lib/test/test_peg_generator/test_c_generator.py‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import io
22
import unittest
3+
from pathlib import PurePath
34
from unittest import mock
45

56
from test import test_tools
@@ -33,7 +34,7 @@ def test_rule_types_distinguish_implicit_and_explicit_void_pointer(self):
3334
explicit[void*]: NAME
3435
""")
3536
generator.rules["explicit"].type = "void *"
36-
start, implicit, explicit = generator.prepare("example.gram").rules
37+
start, implicit, explicit = generator.prepare(PurePath("example.gram")).rules
3738

3839
self.assertIsNone(implicit.signature.return_type)
3940
self.assertEqual(explicit.signature.return_type, "void *")
@@ -49,7 +50,7 @@ def test_rule_types_distinguish_implicit_and_explicit_void_pointer(self):
4950
self.assertEqual(call.function, "_PyPegen_lookahead")
5051
generator = self.make_generator("start: &explicit\nexplicit[void*]: NAME\n")
5152
with self.assertRaisesRegex(RuntimeError, "return type is incompatible"):
52-
generator.prepare("example.gram")
53+
generator.prepare(PurePath("example.gram"))
5354

5455
def test_parser_plan_does_not_depend_on_compilation_state(self):
5556
generator = self.make_generator("""
@@ -63,7 +64,7 @@ def test_parser_plan_does_not_depend_on_compilation_state(self):
6364
expr[expr_ty]: expr '+' NAME | NAME
6465
""")
6566
generator.debug = True
66-
parser = generator.prepare("some/path/example.gram")
67+
parser = generator.prepare(PurePath("some/path/example.gram"))
6768
expected = self.emit_parser(parser)
6869

6970
self.assertEqual(generator.file.getvalue(), "")
@@ -101,12 +102,12 @@ def test_repeated_preparation_keeps_variable_names_local(self):
101102
other: name_var=NUMBER name_var=NAME { name_var_1 }
102103
"""
103104
generator = self.make_generator(source)
104-
parser = generator.prepare("example.gram")
105+
parser = generator.prepare(PurePath("example.gram"))
105106
expected = self.emit_parser(parser)
106107

107108
self.assertEqual(self.emit_parser(parser), expected)
108-
self.assertEqual(generator.prepare("example.gram"), parser)
109-
other = self.make_generator(source).prepare("example.gram")
109+
self.assertEqual(generator.prepare(PurePath("example.gram")), parser)
110+
other = self.make_generator(source).prepare(PurePath("example.gram"))
110111
self.assertEqual(other, parser)
111112
self.assertEqual(self.emit_parser(other), expected)
112113
self.assertEqual(expected.count("expr_ty name_var_1;"), 2)
@@ -118,15 +119,15 @@ def test_prepared_prefixes_preserve_reuse_and_repeatability(self):
118119
start: prefix ':' NAME | prefix ':' NUMBER | NAME | prefix '=' NAME
119120
prefix[expr_ty] (memo): NAME
120121
""")
121-
parser = generator.prepare("example.gram")
122+
parser = generator.prepare(PurePath("example.gram"))
122123
start = parser.rules[0]
123124
prefix, = start.prefixes
124125
self.assertEqual(prefix.type, "expr_ty")
125126
for alt in start.alternatives[:2]:
126127
self.assertIn("!p->call_invalid_rules", alt.calls[0].expression())
127128
self.assertIn(prefix.result, alt.calls[0].expression())
128129
self.assertEqual(start.alternatives[3].calls[0].expression(), "prefix_rule(p)")
129-
self.assertEqual(generator.prepare("example.gram"), parser)
130+
self.assertEqual(generator.prepare(PurePath("example.gram")), parser)
130131
expected = self.emit_parser(parser)
131132
generator.rules.clear()
132133
generator.all_rules.clear()
@@ -137,7 +138,7 @@ def test_nullable_prefix_is_not_reused(self):
137138
start: prefix ':' NAME | prefix ':' NUMBER
138139
prefix (memo): NAME?
139140
""")
140-
start = generator.prepare("example.gram").rules[0]
141+
start = generator.prepare(PurePath("example.gram")).rules[0]
141142
self.assertEqual(start.prefixes, ())
142143
for alt in start.alternatives:
143144
self.assertEqual(alt.calls[0].expression(), "prefix_rule(p)")
@@ -148,11 +149,11 @@ def test_invalid_trailer_fails_before_output(self):
148149
start: NAME ENDMARKER
149150
""")
150151
with self.assertRaisesRegex(KeyError, "missing"):
151-
generator.generate("example.gram")
152+
generator.generate(PurePath("example.gram"))
152153
self.assertEqual(generator.file.getvalue(), "")
153154

154155
def test_empty_keyword_tables(self):
155-
parser = self.make_generator("start: NAME ENDMARKER\n").prepare("example.gram")
156+
parser = self.make_generator("start: NAME ENDMARKER\n").prepare(PurePath("example.gram"))
156157
source = self.emit_parser(parser)
157158

158159
self.assertEqual(parser.keyword_groups, ())
@@ -195,9 +196,9 @@ def test_helper_resolution_does_not_depend_on_display_settings(self):
195196
grammar_module, "SIMPLE_STR", simple
196197
):
197198
generator = self.make_generator(source)
198-
expected = generator.prepare("example.gram").rules
199+
expected = generator.prepare(PurePath("example.gram")).rules
199200
with mock.patch.object(grammar_module, "SIMPLE_STR", not simple):
200-
actual = generator.prepare("example.gram").rules
201+
actual = generator.prepare(PurePath("example.gram")).rules
201202
self.assertEqual(len(actual), len(expected))
202203
for old, new in zip(expected, actual):
203204
self.assertEqual(old.signature, new.signature)
@@ -232,7 +233,7 @@ def test_invalid_rule_gating_uses_references(self):
232233
start: {item} {{ _PyPegen_dummy_name(p) }}
233234
invalid_example: NAME
234235
""")
235-
start = generator.prepare("example.gram").rules[0]
236+
start = generator.prepare(PurePath("example.gram")).rules[0]
236237
self.assertEqual(
237238
start.alternatives[0].requires_invalid_rules,
238239
requires_invalid_rules,
@@ -243,7 +244,7 @@ def test_lowering_preserves_legacy_named_call_types(self):
243244
start: Mixed LPAR ENDMARKER
244245
Mixed[expr_ty]: NAME
245246
""")
246-
start, mixed = generator.prepare("example.gram").rules
247+
start, mixed = generator.prepare(PurePath("example.gram")).rules
247248
self.assertEqual(mixed.signature.return_type, "expr_ty")
248249
for call, name in zip(start.alternatives[0].calls, ("Mixed", "LPAR")):
249250
with self.subTest(name=name):

‎Lib/test/test_peg_generator/test_pegen.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io
44
import textwrap
55
import unittest
6+
from pathlib import PurePath
67

78
from test import test_tools
89
from typing import Dict, Any
@@ -573,7 +574,7 @@ def test_mutually_left_recursive(self) -> None:
573574
self.assertFalse(rules["start"].left_recursive)
574575
self.assertTrue(rules["foo"].left_recursive)
575576
self.assertTrue(rules["bar"].left_recursive)
576-
genr.generate("<string>")
577+
genr.generate(PurePath("<string>"))
577578
ns: Dict[str, Any] = {}
578579
exec(out.getvalue(), ns)
579580
parser_class: Type[Parser] = ns["GeneratedParser"]
@@ -670,7 +671,7 @@ def test_nasty_mutually_left_recursive(self) -> None:
670671
grammar: Grammar = parse_string(grammar_source, GrammarParser)
671672
out = io.StringIO()
672673
genr = PythonParserGenerator(grammar, out)
673-
genr.generate("<string>")
674+
genr.generate(PurePath("<string>"))
674675
ns: Dict[str, Any] = {}
675676
exec(out.getvalue(), ns)
676677
parser_class = ns["GeneratedParser"]
@@ -902,7 +903,7 @@ def test_unreachable_explicit(self) -> None:
902903
genr = PythonParserGenerator(
903904
grammar, out, unreachable_formatting="This is a test"
904905
)
905-
genr.generate("<string>")
906+
genr.generate(PurePath("<string>"))
906907
self.assertIn("This is a test", out.getvalue())
907908

908909
def test_unreachable_implicit1(self) -> None:
@@ -915,7 +916,7 @@ def test_unreachable_implicit1(self) -> None:
915916
genr = PythonParserGenerator(
916917
grammar, out, unreachable_formatting="This is a test"
917918
)
918-
genr.generate("<string>")
919+
genr.generate(PurePath("<string>"))
919920
self.assertIn("This is a test", out.getvalue())
920921

921922
def test_unreachable_implicit2(self) -> None:
@@ -928,7 +929,7 @@ def test_unreachable_implicit2(self) -> None:
928929
genr = PythonParserGenerator(
929930
grammar, out, unreachable_formatting="This is a test"
930931
)
931-
genr.generate("<string>")
932+
genr.generate(PurePath("<string>"))
932933
self.assertIn("This is a test", out.getvalue())
933934

934935
def test_unreachable_implicit3(self) -> None:
@@ -941,7 +942,7 @@ def test_unreachable_implicit3(self) -> None:
941942
genr = PythonParserGenerator(
942943
grammar, out, unreachable_formatting="This is a test"
943944
)
944-
genr.generate("<string>")
945+
genr.generate(PurePath("<string>"))
945946
self.assertNotIn("This is a test", out.getvalue())
946947

947948
def test_locations_in_alt_action_and_group(self) -> None:

‎Tools/peg_generator/pegen/__main__.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"""
77

88
import argparse
9+
import pathlib
910
import sys
1011
import time
1112
import token
@@ -87,10 +88,15 @@ def generate_python_code(
8788

8889
c_parser = subparsers.add_parser("c", help="Generate C code for inclusion into CPython")
8990
c_parser.set_defaults(func=generate_c_code)
90-
c_parser.add_argument("grammar_filename", help="Grammar description")
91-
c_parser.add_argument("tokens_filename", help="Tokens description")
91+
c_parser.add_argument("grammar_filename", type=pathlib.Path, help="Grammar description")
92+
c_parser.add_argument("tokens_filename", type=pathlib.Path, help="Tokens description")
9293
c_parser.add_argument(
93-
"-o", "--output", metavar="OUT", default="parse.c", help="Where to write the generated parser"
94+
"-o",
95+
"--output",
96+
metavar="OUT",
97+
type=pathlib.Path,
98+
default=pathlib.Path("parse.c"),
99+
help="Where to write the generated parser",
94100
)
95101
c_parser.add_argument(
96102
"--compile-extension",
@@ -111,12 +117,13 @@ def generate_python_code(
111117
help="Generate Python code, needs grammar definition with Python actions",
112118
)
113119
python_parser.set_defaults(func=generate_python_code)
114-
python_parser.add_argument("grammar_filename", help="Grammar description")
120+
python_parser.add_argument("grammar_filename", type=pathlib.Path, help="Grammar description")
115121
python_parser.add_argument(
116122
"-o",
117123
"--output",
118124
metavar="OUT",
119-
default="parse.py",
125+
type=pathlib.Path,
126+
default=pathlib.Path("parse.py"),
120127
help="Where to write the generated parser",
121128
)
122129
python_parser.add_argument(

‎Tools/peg_generator/pegen/build.py‎

Lines changed: 41 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def fixup_build_ext(cmd: Incomplete) -> None:
7070

7171

7272
def compile_c_extension(
73-
generated_source_path: str,
73+
generated_source_path: pathlib.Path,
7474
build_dir: str | None = None,
7575
verbose: bool = False,
7676
keep_asserts: bool = True,
@@ -101,8 +101,7 @@ def compile_c_extension(
101101
if verbose:
102102
setuptools.logging.set_threshold(logging.DEBUG)
103103

104-
source_file_path = pathlib.Path(generated_source_path)
105-
extension_name = source_file_path.stem
104+
extension_name = generated_source_path.stem
106105
extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
107106
extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
108107
# Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
@@ -122,39 +121,39 @@ def compile_c_extension(
122121
extra_link_args.append("-fno-lto")
123122

124123
common_sources = [
125-
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
126-
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
127-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
128-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"),
129-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
130-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
131-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
132-
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"),
133-
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c"),
134-
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
135-
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
136-
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
137-
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c"),
138-
str(MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c"),
139-
str(MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c"),
140-
str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
124+
MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c",
125+
MOD_DIR.parent.parent.parent / "Python" / "asdl.c",
126+
MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c",
127+
MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c",
128+
MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c",
129+
MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c",
130+
MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c",
131+
MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c",
132+
MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c",
133+
MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c",
134+
MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c",
135+
MOD_DIR.parent.parent.parent / "Parser" / "pegen.c",
136+
MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c",
137+
MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c",
138+
MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c",
139+
MOD_DIR.parent / "peg_extension" / "peg_extension.c",
141140
]
142141
include_dirs = [
143-
str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
144-
str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"),
145-
str(MOD_DIR.parent.parent.parent / "Parser"),
146-
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"),
147-
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"),
142+
MOD_DIR.parent.parent.parent / "Include" / "internal",
143+
MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc",
144+
MOD_DIR.parent.parent.parent / "Parser",
145+
MOD_DIR.parent.parent.parent / "Parser" / "lexer",
146+
MOD_DIR.parent.parent.parent / "Parser" / "tokenizer",
148147
]
149-
library_dirs: list[str] = []
148+
library_dirs: list[pathlib.Path] = []
150149
if sys.platform == "win32":
151150
# HACK: The location of pyconfig.h has moved within our build, and
152151
# setuptools hasn't updated for it yet. So add the path manually for now
153-
include_dirs.append(str(pathlib.Path(sysconfig.get_config_h_filename()).parent))
152+
include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent)
154153
if sysconfig.is_python_build():
155154
# HACK: Our output directory for free-threaded builds has moved, and so
156155
# tests running in-tree require our sys.executable directory for libs
157-
library_dirs.append(str(pathlib.Path(sys.executable).parent))
156+
library_dirs.append(pathlib.Path(sys.executable).parent)
158157
sources = [generated_source_path]
159158
extension = Extension(
160159
extension_name,
@@ -166,9 +165,9 @@ def compile_c_extension(
166165
cmd = dist.get_command_obj("build_ext")
167166
assert isinstance(cmd, setuptools.command.build_ext.build_ext)
168167
fixup_build_ext(cmd)
169-
cmd.build_lib = str(source_file_path.parent)
170-
cmd.include_dirs = include_dirs
171-
cmd.library_dirs = library_dirs
168+
cmd.build_lib = str(generated_source_path.parent)
169+
cmd.include_dirs = [str(d) for d in include_dirs]
170+
cmd.library_dirs = [str(d) for d in library_dirs]
172171
if build_dir:
173172
cmd.build_temp = build_dir
174173
cmd.ensure_finalized()
@@ -247,15 +246,15 @@ def compile_c_extension(
247246

248247

249248
def build_parser(
250-
grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False
249+
grammar_file: pathlib.Path, verbose_tokenizer: bool = False, verbose_parser: bool = False
251250
) -> tuple[Grammar, Parser, Tokenizer]:
252251
with open(grammar_file) as file:
253252
tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
254253
parser = GrammarParser(tokenizer, verbose=verbose_parser)
255254
grammar = parser.start()
256255

257256
if not grammar:
258-
raise parser.make_syntax_error(grammar_file)
257+
raise parser.make_syntax_error(str(grammar_file))
259258

260259
return grammar, parser, tokenizer
261260

@@ -291,9 +290,9 @@ def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
291290

292291
def build_c_generator(
293292
grammar: Grammar,
294-
grammar_file: str,
295-
tokens_file: str,
296-
output_file: str,
293+
grammar_file: pathlib.Path,
294+
tokens_file: pathlib.Path,
295+
output_file: pathlib.Path,
297296
compile_extension: bool = False,
298297
verbose_c_extension: bool = False,
299298
keep_asserts_in_extension: bool = True,
@@ -320,8 +319,8 @@ def build_c_generator(
320319

321320
def build_python_generator(
322321
grammar: Grammar,
323-
grammar_file: str,
324-
output_file: str,
322+
grammar_file: pathlib.Path,
323+
output_file: pathlib.Path,
325324
skip_actions: bool = False,
326325
) -> ParserGenerator:
327326
with open(output_file, "w") as file:
@@ -331,9 +330,9 @@ def build_python_generator(
331330

332331

333332
def build_c_parser_and_generator(
334-
grammar_file: str,
335-
tokens_file: str,
336-
output_file: str,
333+
grammar_file: pathlib.Path,
334+
tokens_file: pathlib.Path,
335+
output_file: pathlib.Path,
337336
compile_extension: bool = False,
338337
verbose_tokenizer: bool = False,
339338
verbose_parser: bool = False,
@@ -375,8 +374,8 @@ def build_c_parser_and_generator(
375374

376375

377376
def build_python_parser_and_generator(
378-
grammar_file: str,
379-
output_file: str,
377+
grammar_file: pathlib.Path,
378+
output_file: pathlib.Path,
380379
verbose_tokenizer: bool = False,
381380
verbose_parser: bool = False,
382381
skip_actions: bool = False,

0 commit comments

Comments
 (0)