From 021dc4f8d35283386953a663e094b04f7f0c4c18 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 26 May 2020 01:48:00 +0300 Subject: [PATCH 1/8] Refactor test_parse_directory in Tools/peg_generator/scripts --- Tools/peg_generator/Makefile | 6 - Tools/peg_generator/scripts/benchmark.py | 3 - .../scripts/test_parse_directory.py | 165 ++++++++---------- .../scripts/test_pypi_packages.py | 13 +- 4 files changed, 75 insertions(+), 112 deletions(-) diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index e7a190c1bcd13d..ada2086c0de156 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -85,8 +85,6 @@ time_old_parse: venv data/xxl.py time_peg_dir: venv $(VENVPYTHON) scripts/test_parse_directory.py \ - --grammar-file $(GRAMMAR) \ - --tokens-file $(TOKENS) \ -d $(TESTDIR) \ $(TESTFLAGS) \ --exclude "*/failset/*" \ @@ -95,12 +93,8 @@ time_peg_dir: venv time_stdlib: $(CPYTHON) venv $(VENVPYTHON) scripts/test_parse_directory.py \ - --grammar-file $(GRAMMAR) \ - --tokens-file $(TOKENS) \ -d $(CPYTHON) \ $(TESTFLAGS) \ - --exclude "*/test2to3/*" \ - --exclude "*/test2to3/**/*" \ --exclude "*/bad*" \ --exclude "*/lib2to3/tests/data/*" diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 71512c22a355b3..78441454433192 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -90,11 +90,8 @@ def run_benchmark_stdlib(subcommand, parser): for _ in range(3): parse_directory( "../../Lib", - "../../Grammar/python.gram", - "../../Grammar/Tokens", verbose=False, excluded_files=["*/bad*", "*/lib2to3/tests/data/*",], - skip_actions=False, tree_arg=0, short=True, mode=2 if subcommand == "compile" else 1, diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index e88afe1539ce10..f41bd202081622 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -11,7 +11,7 @@ from glob import glob from pathlib import PurePath -from typing import List, Optional, Any +from typing import List, Optional, Any, Tuple sys.path.insert(0, os.getcwd()) from pegen.ast_dump import ast_dump @@ -27,8 +27,6 @@ description="Helper program to test directories or files for pegen", ) argparser.add_argument("-d", "--directory", help="Directory path containing files to test") -argparser.add_argument("--grammar-file", help="Grammar file path") -argparser.add_argument("--tokens-file", help="Tokens file path") argparser.add_argument( "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude" ) @@ -38,9 +36,6 @@ argparser.add_argument( "-v", "--verbose", action="store_true", help="Display detailed errors for failures" ) -argparser.add_argument( - "--skip-actions", action="store_true", help="Suppress code emission for rule actions", -) argparser.add_argument( "-t", "--tree", action="count", help="Compare parse tree to official AST", default=0 ) @@ -113,92 +108,35 @@ def compare_trees( return 1 -def parse_directory( - directory: str, - grammar_file: str, - tokens_file: str, - verbose: bool, - excluded_files: List[str], - skip_actions: bool, - tree_arg: int, - short: bool, - mode: int, - parser: str, -) -> int: - if parser == "cpython" and (tree_arg or mode == 0): - print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) - return 1 - - if not directory: - print("You must specify a directory of files to test.", file=sys.stderr) - return 1 - - if grammar_file and tokens_file: - if not os.path.exists(grammar_file): - print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) - return 1 +def parse_file(source: str, file: str, mode: int, parser: str) -> Tuple[Any, float]: + t0 = time.time() + if mode == 2: + result = _peg_parser.compile_string( + source, + filename=file, + oldparser=parser == "cpython", + ) else: - print( - "A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n" + result = _peg_parser.parse_string( + source, + filename=file, + oldparser=parser == "cpython" ) + t1 = time.time() + return result, t1 - t0 - if tree_arg: - assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" - # For a given directory, traverse files and attempt to parse each one - # - Output success/failure for each file - errors = 0 - files = [] - trees = {} # Trees to compare (after everything else is done) - total_seconds = 0 +def is_parsing_failure(source: str) -> bool: + try: + _peg_parser.parse_string(source, mode="exec", oldparser=True) + except SyntaxError: + return False + else: + return True - for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): - # Only attempt to parse Python files and files that are not excluded - should_exclude_file = False - for pattern in excluded_files: - if PurePath(file).match(pattern): - should_exclude_file = True - break - - if not should_exclude_file: - with tokenize.open(file) as f: - source = f.read() - try: - t0 = time.time() - if mode == 2: - result = _peg_parser.compile_string( - source, - filename=file, - oldparser=parser == "cpython", - ) - else: - result = _peg_parser.parse_string( - source, - filename=file, - oldparser=parser == "cpython" - ) - t1 = time.time() - total_seconds += (t1 - t0) - if tree_arg: - trees[file] = result - if not short: - report_status(succeeded=True, file=file, verbose=verbose) - except Exception as error: - try: - _peg_parser.parse_string(source, mode="exec", oldparser=True) - except Exception: - if not short: - print(f"File {file} cannot be parsed by either pegen or the ast module.") - else: - report_status( - succeeded=False, file=file, verbose=verbose, error=error, short=short - ) - errors += 1 - files.append(file) - t1 = time.time() +def generate_time_stats(files, total_seconds) -> None: total_files = len(files) - total_bytes = 0 total_lines = 0 for file in files: @@ -217,6 +155,57 @@ def parse_directory( f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) + +def parse_directory( + directory: str, + verbose: bool, + excluded_files: List[str], + tree_arg: int, + short: bool, + mode: int, + parser: str, +) -> int: + if tree_arg: + assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" + + if parser == "cpython" and tree_arg: + print("Cannot specify tree argument with the cpython parser.", file=sys.stderr) + return 1 + + # For a given directory, traverse files and attempt to parse each one + # - Output success/failure for each file + errors = 0 + files = [] + trees = {} # Trees to compare (after everything else is done) + total_seconds = 0 + + for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): + # Only attempt to parse Python files and files that are not excluded + if any(PurePath(file).match(pattern) for pattern in excluded_files): + continue + + with tokenize.open(file) as f: + source = f.read() + + try: + result, dt = parse_file(source, file, mode, parser) + total_seconds += dt + if tree_arg: + trees[file] = result + report_status(succeeded=True, file=file, verbose=verbose, short=short) + except SyntaxError as error: + if is_parsing_failure(source): + print(f"File {file} cannot be parsed by either parser.") + else: + report_status( + succeeded=False, file=file, verbose=verbose, error=error, short=short + ) + errors += 1 + files.append(file) + + t1 = time.time() + + generate_time_stats(files, total_seconds) if short: print_memstats() @@ -240,22 +229,16 @@ def parse_directory( def main() -> None: args = argparser.parse_args() directory = args.directory - grammar_file = args.grammar_file - tokens_file = args.tokens_file verbose = args.verbose excluded_files = args.exclude - skip_actions = args.skip_actions tree = args.tree short = args.short mode = 1 if args.tree else 2 sys.exit( parse_directory( directory, - grammar_file, - tokens_file, verbose, excluded_files, - skip_actions, tree, short, mode, diff --git a/Tools/peg_generator/scripts/test_pypi_packages.py b/Tools/peg_generator/scripts/test_pypi_packages.py index 98f77785cdd1c6..c8a2992ce6d8df 100755 --- a/Tools/peg_generator/scripts/test_pypi_packages.py +++ b/Tools/peg_generator/scripts/test_pypi_packages.py @@ -57,19 +57,8 @@ def find_dirname(package_name: str) -> str: def run_tests(dirname: str, tree: int) -> int: return test_parse_directory.parse_directory( dirname, - HERE / ".." / ".." / ".." / "Grammar" / "python.gram", - HERE / ".." / ".." / ".." / "Grammar" / "Tokens", verbose=False, - excluded_files=[ - "*/failset/*", - "*/failset/**", - "*/failset/**/*", - "*/test2to3/*", - "*/test2to3/**/*", - "*/bad*", - "*/lib2to3/tests/data/*", - ], - skip_actions=False, + excluded_files=[], tree_arg=tree, short=True, mode=1, From f07d11b1d3bf3a51f8f36a0335e8127be4a68b8a Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 26 May 2020 14:06:54 +0300 Subject: [PATCH 2/8] Address feedback; refactor parser arg to be either 'new' or 'old' --- Tools/peg_generator/Makefile | 8 ++++---- Tools/peg_generator/scripts/benchmark.py | 8 ++++---- Tools/peg_generator/scripts/grammar_grapher.py | 6 +++--- Tools/peg_generator/scripts/show_parse.py | 17 +++++++++++------ .../scripts/test_parse_directory.py | 14 +++++++------- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index ada2086c0de156..fb727c048b3118 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -70,18 +70,18 @@ stats: peg_extension/parse.c data/xxl.py time: time_compile time_compile: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl compile + $(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl compile time_parse: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl parse + $(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl parse time_old: time_old_compile time_old_compile: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl compile + $(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl compile time_old_parse: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl parse + $(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl parse time_peg_dir: venv $(VENVPYTHON) scripts/test_parse_directory.py \ diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 78441454433192..d82854b392b313 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -24,7 +24,7 @@ argparser.add_argument( "--parser", action="store", - choices=["pegen", "cpython"], + choices=["new", "old"], default="pegen", help="Which parser to benchmark (default is pegen)", ) @@ -62,7 +62,7 @@ def wrapper(*args): @benchmark def time_compile(source, parser): - if parser == "cpython": + if parser == "old": return _peg_parser.compile_string( source, oldparser=True, @@ -73,7 +73,7 @@ def time_compile(source, parser): @benchmark def time_parse(source, parser): - if parser == "cpython": + if parser == "old": return _peg_parser.parse_string(source, oldparser=True) else: return _peg_parser.parse_string(source) @@ -95,7 +95,7 @@ def run_benchmark_stdlib(subcommand, parser): tree_arg=0, short=True, mode=2 if subcommand == "compile" else 1, - parser=parser, + oldparser=parser == "old", ) diff --git a/Tools/peg_generator/scripts/grammar_grapher.py b/Tools/peg_generator/scripts/grammar_grapher.py index 3aa25466c70d47..f5e623c1db487a 100755 --- a/Tools/peg_generator/scripts/grammar_grapher.py +++ b/Tools/peg_generator/scripts/grammar_grapher.py @@ -91,11 +91,11 @@ def main() -> None: references[name] = set(references_for_item(rule)) # Flatten the start node if has only a single reference - root_node = "start" - if start := references["start"]: + root_node = "file" # The start rule is called 'file' now + if start := references["file"]: if len(start) == 1: root_node = list(start)[0] - del references["start"] + del references["file"] print("digraph g1 {") print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps diff --git a/Tools/peg_generator/scripts/show_parse.py b/Tools/peg_generator/scripts/show_parse.py index 1c1996f40f74e0..6c2c08f278caa1 100755 --- a/Tools/peg_generator/scripts/show_parse.py +++ b/Tools/peg_generator/scripts/show_parse.py @@ -41,7 +41,13 @@ parser.add_argument( "-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)" ) -parser.add_argument("-g", "--grammar-file", help="grammar to use (default: use the ast module)") +parser.add_argument( + "-p", + "--parser", + choices=["new", "old"], + default="new", + help="choose the parser to use" +) parser.add_argument( "-m", "--multiline", @@ -84,14 +90,13 @@ def print_parse(source: str, verbose: bool = False) -> None: def main() -> None: args = parser.parse_args() - if args.diff and not args.grammar_file: - parser.error("-d/--diff requires -g/--grammar-file") + new_parser = args.parser == "new" if args.multiline: sep = "\n" else: sep = " " program = sep.join(args.program) - if args.grammar_file: + if new_parser: tree = _peg_parser.parse_string(program) if args.diff: @@ -104,11 +109,11 @@ def main() -> None: else: print("# Trees are the same") else: - print(f"# Parsed using {args.grammar_file}") + print(f"# Parsed using the new parser") print(format_tree(tree, args.verbose)) else: tree = _peg_parser.parse_string(program, oldparser=True) - print("# Parse using the old parser") + print("# Parsed using the old parser") print(format_tree(tree, args.verbose)) diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index f41bd202081622..6b134975faf5ad 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -108,19 +108,19 @@ def compare_trees( return 1 -def parse_file(source: str, file: str, mode: int, parser: str) -> Tuple[Any, float]: +def parse_file(source: str, file: str, mode: int, oldparser: bool) -> Tuple[Any, float]: t0 = time.time() if mode == 2: result = _peg_parser.compile_string( source, filename=file, - oldparser=parser == "cpython", + oldparser=oldparser, ) else: result = _peg_parser.parse_string( source, filename=file, - oldparser=parser == "cpython" + oldparser=oldparser, ) t1 = time.time() return result, t1 - t0 @@ -163,12 +163,12 @@ def parse_directory( tree_arg: int, short: bool, mode: int, - parser: str, + oldparser: bool, ) -> int: if tree_arg: assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" - if parser == "cpython" and tree_arg: + if oldparser and tree_arg: print("Cannot specify tree argument with the cpython parser.", file=sys.stderr) return 1 @@ -188,7 +188,7 @@ def parse_directory( source = f.read() try: - result, dt = parse_file(source, file, mode, parser) + result, dt = parse_file(source, file, mode, oldparser) total_seconds += dt if tree_arg: trees[file] = result @@ -242,7 +242,7 @@ def main() -> None: tree, short, mode, - "pegen", + False, ) ) From d0b2b124843e94a7e25391e3183d79df05e014c6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 26 May 2020 23:01:44 +0300 Subject: [PATCH 3/8] Update Tools/peg_generator/scripts/test_parse_directory.py Co-authored-by: Pablo Galindo --- Tools/peg_generator/scripts/test_parse_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index 6b134975faf5ad..b00123405e9d4d 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -242,7 +242,7 @@ def main() -> None: tree, short, mode, - False, + oldparser=False, ) ) From 04edae0573cb03cfdaece97a5fa348b2bbfa8efe Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 26 May 2020 23:01:53 +0300 Subject: [PATCH 4/8] Update Tools/peg_generator/scripts/benchmark.py Co-authored-by: Pablo Galindo --- Tools/peg_generator/scripts/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index d82854b392b313..aa205005c986a7 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -95,7 +95,7 @@ def run_benchmark_stdlib(subcommand, parser): tree_arg=0, short=True, mode=2 if subcommand == "compile" else 1, - oldparser=parser == "old", + oldparser=(parser == "old"), ) From f0f1d1693350befc998bb68c3fdb26cbf2924af3 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 26 May 2020 23:45:00 +0300 Subject: [PATCH 5/8] Add start cmd arg to grammar grapher, don't flatten root node and fix dot syntax error --- .../peg_generator/scripts/grammar_grapher.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Tools/peg_generator/scripts/grammar_grapher.py b/Tools/peg_generator/scripts/grammar_grapher.py index f5e623c1db487a..4afdbce8f966f2 100755 --- a/Tools/peg_generator/scripts/grammar_grapher.py +++ b/Tools/peg_generator/scripts/grammar_grapher.py @@ -42,6 +42,13 @@ ) argparser = argparse.ArgumentParser(prog="graph_grammar", description="Graph a grammar tree",) +argparser.add_argument( + "-s", + "--start", + choices=["exec", "eval", "single"], + default="exec", + help="Choose the grammar's start rule (exec, eval or single)", +) argparser.add_argument("grammar_file", help="The grammar file to graph") @@ -91,19 +98,15 @@ def main() -> None: references[name] = set(references_for_item(rule)) # Flatten the start node if has only a single reference - root_node = "file" # The start rule is called 'file' now - if start := references["file"]: - if len(start) == 1: - root_node = list(start)[0] - del references["file"] + root_node = {"exec": "file", "eval": "eval", "single": "interactive"}[args.start] print("digraph g1 {") print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps print(f'\troot="{root_node}";') - print(f"\t{root_node} [color=green, shape=circle]") + print(f"\t{root_node} [color=green, shape=circle];") for name, refs in references.items(): - if refs: # Ignore empty sets - print(f"\t{name} -> {','.join(refs)};") + for ref in refs: + print(f"\t{name} -> {ref};") print("}") From 3e605a09ee5d8c623b648e65cd270f3b06562aa7 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 27 May 2020 00:37:59 +0300 Subject: [PATCH 6/8] Update Tools/peg_generator/scripts/test_parse_directory.py Co-authored-by: Pablo Galindo --- Tools/peg_generator/scripts/test_parse_directory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index b00123405e9d4d..193a092934b604 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -131,8 +131,7 @@ def is_parsing_failure(source: str) -> bool: _peg_parser.parse_string(source, mode="exec", oldparser=True) except SyntaxError: return False - else: - return True + return True def generate_time_stats(files, total_seconds) -> None: From 2a02868b751d41a8e224f8cc5d07769ea843b3ef Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 6 Jun 2020 03:13:53 +0300 Subject: [PATCH 7/8] Address feedback; Add ast kwarg to _peg_parser.parse_string --- Modules/_peg_parser.c | 16 +++++++++++---- Tools/peg_generator/scripts/benchmark.py | 20 +++++++++++++++++-- Tools/peg_generator/scripts/show_parse.py | 6 +++--- .../scripts/test_parse_directory.py | 9 +++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c index b66d5a83a84f64..ca2a3cf7b5fd80 100644 --- a/Modules/_peg_parser.c +++ b/Modules/_peg_parser.c @@ -80,14 +80,15 @@ _Py_compile_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL}; + static char *keywords[] = {"string", "filename", "mode", "oldparser", "ast", NULL}; char *the_string; char *filename = ""; char *mode_str = "exec"; int oldparser = 0; + int ast = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords, - &the_string, &filename, &mode_str, &oldparser)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sspp", keywords, + &the_string, &filename, &mode_str, &oldparser, &ast)) { return NULL; } @@ -110,7 +111,14 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *result = PyAST_mod2obj(mod); + PyObject *result; + if (ast) { + result = PyAST_mod2obj(mod); + } + else { + Py_INCREF(Py_None); + result = Py_None; + } PyArena_Free(arena); return result; } diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index aa205005c986a7..af356bed78391d 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -40,7 +40,12 @@ command_compile = subcommands.add_parser( "compile", help="Benchmark parsing and compiling to bytecode" ) -command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST") +command_parse = subcommands.add_parser( + "parse", help="Benchmark parsing and generating an ast.AST" +) +command_notree = subcommands.add_parser( + "notree", help="Benchmark parsing and dumping the tree" +) def benchmark(func): @@ -79,14 +84,25 @@ def time_parse(source, parser): return _peg_parser.parse_string(source) +@benchmark +def time_notree(source, parser): + if parser == "old": + return _peg_parser.parse_string(source, oldparser=True, ast=False) + else: + return _peg_parser.parse_string(source, ast=False) + + def run_benchmark_xxl(subcommand, parser, source): if subcommand == "compile": time_compile(source, parser) elif subcommand == "parse": time_parse(source, parser) + elif subcommand == "notree": + time_notree(source, parser) def run_benchmark_stdlib(subcommand, parser): + modes = {"compile": 2, "parse": 1, "notree": 0} for _ in range(3): parse_directory( "../../Lib", @@ -94,7 +110,7 @@ def run_benchmark_stdlib(subcommand, parser): excluded_files=["*/bad*", "*/lib2to3/tests/data/*",], tree_arg=0, short=True, - mode=2 if subcommand == "compile" else 1, + mode=modes[subcommand], oldparser=(parser == "old"), ) diff --git a/Tools/peg_generator/scripts/show_parse.py b/Tools/peg_generator/scripts/show_parse.py index 6c2c08f278caa1..b4ee5a1b357f74 100755 --- a/Tools/peg_generator/scripts/show_parse.py +++ b/Tools/peg_generator/scripts/show_parse.py @@ -100,8 +100,8 @@ def main() -> None: tree = _peg_parser.parse_string(program) if args.diff: - a = tree - b = _peg_parser.parse_string(program, oldparser=True) + a = _peg_parser.parse_string(program, oldparser=True) + b = tree diff = diff_trees(a, b, args.verbose) if diff: for line in diff: @@ -109,7 +109,7 @@ def main() -> None: else: print("# Trees are the same") else: - print(f"# Parsed using the new parser") + print("# Parsed using the new parser") print(format_tree(tree, args.verbose)) else: tree = _peg_parser.parse_string(program, oldparser=True) diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index 193a092934b604..63204ce9dc1934 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -22,6 +22,10 @@ FAIL = "\033[91m" ENDC = "\033[0m" +COMPILE = 2 +PARSE = 1 +NOTREE = 0 + argparser = argparse.ArgumentParser( prog="test_parse_directory", description="Helper program to test directories or files for pegen", @@ -110,7 +114,7 @@ def compare_trees( def parse_file(source: str, file: str, mode: int, oldparser: bool) -> Tuple[Any, float]: t0 = time.time() - if mode == 2: + if mode == COMPILE: result = _peg_parser.compile_string( source, filename=file, @@ -121,6 +125,7 @@ def parse_file(source: str, file: str, mode: int, oldparser: bool) -> Tuple[Any, source, filename=file, oldparser=oldparser, + ast=(mode == PARSE), ) t1 = time.time() return result, t1 - t0 @@ -165,7 +170,7 @@ def parse_directory( oldparser: bool, ) -> int: if tree_arg: - assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" + assert mode == PARSE, "Mode should be 1 (parse), when comparing the generated trees" if oldparser and tree_arg: print("Cannot specify tree argument with the cpython parser.", file=sys.stderr) From 2a22c220448ad5f3f383459658b6dc8502795d50 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 6 Jun 2020 03:23:03 +0300 Subject: [PATCH 8/8] Pass correct mode to parse_directory in test_pypi_packages.py --- Tools/peg_generator/scripts/test_pypi_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/scripts/test_pypi_packages.py b/Tools/peg_generator/scripts/test_pypi_packages.py index c8a2992ce6d8df..f014753b3cd23e 100755 --- a/Tools/peg_generator/scripts/test_pypi_packages.py +++ b/Tools/peg_generator/scripts/test_pypi_packages.py @@ -61,7 +61,7 @@ def run_tests(dirname: str, tree: int) -> int: excluded_files=[], tree_arg=tree, short=True, - mode=1, + mode=1 if tree else 0, parser="pegen", )