From 2b755f40ba3a53dad022e07b8a96c4191e37d06d Mon Sep 17 00:00:00 2001 From: isidentical Date: Tue, 21 Jan 2020 16:38:36 +0300 Subject: [PATCH 01/10] bpo-39411: pyclbr rewrite on AST --- Doc/library/pyclbr.rst | 7 + Lib/pyclbr.py | 324 +++++++----------- Lib/test/pyclbr_input.py | 7 +- Lib/test/test_pyclbr.py | 34 +- .../2020-01-21-16-38-25.bpo-39411.9uHFqT.rst | 3 + 5 files changed, 152 insertions(+), 223 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index b80a2faed9b424..15f28c1f52e868 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -94,6 +94,13 @@ statements. They have the following attributes: .. versionadded:: 3.7 +.. attribute:: Function.parent + + For functions that are defined with ``async`` prefix, ``True``. + + .. versionadded:: 3.9 + + .. _pyclbr-class-objects: Class Objects diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 99a17343fb61fd..86119d83ecfa7d 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -25,7 +25,9 @@ children -- nested objects contained in this object. The 'children' attribute is a dictionary mapping names to objects. -Instances of Function describe functions with the attributes from _Object. +Instances of Function describe functions with the attributes from _Object, +plus the following: + is_async -- if function defined with 'async' prefix Instances of Class describe classes with the attributes from _Object, plus the following: @@ -38,11 +40,10 @@ shouldn't happen often. """ -import io +import ast +import copy import sys import importlib.util -import tokenize -from token import NAME, DEDENT, OP __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] @@ -58,6 +59,8 @@ def __init__(self, module, name, file, lineno, parent): self.lineno = lineno self.parent = parent self.children = {} + if parent is not None: + parent._addchild(name, self) def _addchild(self, name, obj): self.children[name] = obj @@ -65,35 +68,22 @@ def _addchild(self, name, obj): class Function(_Object): "Information about a Python function, including methods." - def __init__(self, module, name, file, lineno, parent=None): - _Object.__init__(self, module, name, file, lineno, parent) - + def __init__(self, module, name, file, lineno, parent=None, is_async=False): + super().__init__(module, name, file, lineno, parent) + self.is_async = is_async + if isinstance(parent, Class): + parent._addmethod(name, lineno) class Class(_Object): "Information about a Python class." - def __init__(self, module, name, super, file, lineno, parent=None): - _Object.__init__(self, module, name, file, lineno, parent) - self.super = [] if super is None else super + def __init__(self, module, name, super_, file, lineno, parent=None): + super().__init__(module, name, file, lineno, parent) + self.super = super_ or [] self.methods = {} def _addmethod(self, name, lineno): self.methods[name] = lineno - -def _nest_function(ob, func_name, lineno): - "Return a Function after nesting within ob." - newfunc = Function(ob.module, func_name, ob.file, lineno, ob) - ob._addchild(func_name, newfunc) - if isinstance(ob, Class): - ob._addmethod(func_name, lineno) - return newfunc - -def _nest_class(ob, class_name, lineno, super=None): - "Return a Class after nesting within ob." - newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) - ob._addchild(class_name, newclass) - return newclass - def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -179,187 +169,113 @@ def _readmodule(module, path, inpackage=None): return _create_tree(fullmodule, path, fname, source, tree, inpackage) -def _create_tree(fullmodule, path, fname, source, tree, inpackage): - """Return the tree for a particular module. - - fullmodule (full module name), inpackage+module, becomes o.module. - path is passed to recursive calls of _readmodule. - fname becomes o.file. - source is tokenized. Imports cause recursive calls to _readmodule. - tree is {} or {'__path__': }. - inpackage, None or string, is passed to recursive calls of _readmodule. - - The effect of recursive calls is mutation of global _modules. - """ - f = io.StringIO(source) +class _ClassBrowser(ast.NodeVisitor): + def __init__(self, module, path, file, tree, inpackage): + self.path = path + self.tree = tree + self.file = file + self.module = module + self.inpackage = inpackage + self.stack = [] + + def visit_ClassDef(self, node): + bases = [] + for base in node.bases: + name = ast.unparse(base) + if name in self.tree: + # We know this super class. + bases.append(self.tree[name]) + elif len(names := name.split(".")) > 1: + # Super class form is module.class: + # look in module for class. + module = names[-2] + class_ = names[-1] + if module in _modules: + bases.append(_modules[module].get(class_) or name) + else: + bases.append(name) + + parent = self.stack[-1] if self.stack else None + class_ = Class( + self.module, node.name, bases, self.file, node.lineno, parent + ) + if parent is None: + self.tree[node.name] = class_ + self.stack.append(class_) + self.generic_visit(node) + self.stack.pop() + + def visit_Assign(self, node): + if ( + not len(node.targets) == 1 + or not len(self.stack) > 0 + or not isinstance(self.stack[-1], Class) + or not isinstance(node.targets[0], ast.Name) + or not isinstance(node.value, ast.Name) + or not isinstance(self.tree.get(node.value.id), Function) + ): + return + + name = node.targets[0].id + child = copy.deepcopy(self.tree[node.value.id]) + child.parent = self.stack[-1] + self.stack[-1]._addchild(name, child) + self.stack[-1]._addmethod(name, node.lineno) + + def visit_FunctionDef(self, node, *, is_async=True): + parent = self.stack[-1] if self.stack else None + function = Function( + self.module, node.name, self.file, node.lineno, parent, is_async + ) + if parent is None: + self.tree[node.name] = function + self.stack.append(function) + self.generic_visit(node) + self.stack.pop() + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node, is_async=True) + + def visit_Import(self, node): + if node.col_offset != 0: + return + + for module in node.names: + try: + try: + _readmodule(module.name, self.path, self.inpackage) + except ImportError: + _readmodule(module.name, []) + except (ImportError, SyntaxError): + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + + def visit_ImportFrom(self, node): + if node.col_offset != 0: + return + try: + module = "." * node.level + if node.module: + module += node.module + module = _readmodule(module, self.path, self.inpackage) + except (ImportError, SyntaxError): + return + + for name in node.names: + if name.name in module: + self.tree[name.asname or name.name] = module[name.name] + elif name.name == "*": + for import_name, import_value in module.items(): + if import_name.startswith("_"): + continue + self.tree[import_name] = import_value - stack = [] # Initialize stack of (class, indent) pairs. - g = tokenize.generate_tokens(f.readline) - try: - for tokentype, token, start, _end, _line in g: - if tokentype == DEDENT: - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - elif token == 'def': - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - tokentype, func_name, start = next(g)[0:3] - if tokentype != NAME: - continue # Skip def with syntax error. - cur_func = None - if stack: - cur_obj = stack[-1][0] - cur_func = _nest_function(cur_obj, func_name, lineno) - else: - # It is just a function. - cur_func = Function(fullmodule, func_name, fname, lineno) - tree[func_name] = cur_func - stack.append((cur_func, thisindent)) - elif token == 'class': - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - tokentype, class_name, start = next(g)[0:3] - if tokentype != NAME: - continue # Skip class with syntax error. - # Parse what follows the class name. - tokentype, token, start = next(g)[0:3] - inherit = None - if token == '(': - names = [] # Initialize list of superclasses. - level = 1 - super = [] # Tokens making up current superclass. - while True: - tokentype, token, start = next(g)[0:3] - if token in (')', ',') and level == 1: - n = "".join(super) - if n in tree: - # We know this super class. - n = tree[n] - else: - c = n.split('.') - if len(c) > 1: - # Super class form is module.class: - # look in module for class. - m = c[-2] - c = c[-1] - if m in _modules: - d = _modules[m] - if c in d: - n = d[c] - names.append(n) - super = [] - if token == '(': - level += 1 - elif token == ')': - level -= 1 - if level == 0: - break - elif token == ',' and level == 1: - pass - # Only use NAME and OP (== dot) tokens for type name. - elif tokentype in (NAME, OP) and level == 1: - super.append(token) - # Expressions in the base list are not supported. - inherit = names - if stack: - cur_obj = stack[-1][0] - cur_class = _nest_class( - cur_obj, class_name, lineno, inherit) - else: - cur_class = Class(fullmodule, class_name, inherit, - fname, lineno) - tree[class_name] = cur_class - stack.append((cur_class, thisindent)) - elif token == 'import' and start[1] == 0: - modules = _getnamelist(g) - for mod, _mod2 in modules: - try: - # Recursively read the imported module. - if inpackage is None: - _readmodule(mod, path) - else: - try: - _readmodule(mod, path, inpackage) - except ImportError: - _readmodule(mod, []) - except: - # If we can't find or parse the imported module, - # too bad -- don't die here. - pass - elif token == 'from' and start[1] == 0: - mod, token = _getname(g) - if not mod or token != "import": - continue - names = _getnamelist(g) - try: - # Recursively read the imported module. - d = _readmodule(mod, path, inpackage) - except: - # If we can't find or parse the imported module, - # too bad -- don't die here. - continue - # Add any classes that were defined in the imported module - # to our name space if they were mentioned in the list. - for n, n2 in names: - if n in d: - tree[n2 or n] = d[n] - elif n == '*': - # Don't add names that start with _. - for n in d: - if n[0] != '_': - tree[n] = d[n] - except StopIteration: - pass - - f.close() - return tree - - -def _getnamelist(g): - """Return list of (dotted-name, as-name or None) tuples for token source g. - - An as-name is the name that follows 'as' in an as clause. - """ - names = [] - while True: - name, token = _getname(g) - if not name: - break - if token == 'as': - name2, token = _getname(g) - else: - name2 = None - names.append((name, name2)) - while token != "," and "\n" not in token: - token = next(g)[1] - if token != ",": - break - return names - - -def _getname(g): - "Return (dotted-name or None, next-token) tuple for token source g." - parts = [] - tokentype, token = next(g)[0:2] - if tokentype != NAME and token != '*': - return (None, token) - parts.append(token) - while True: - tokentype, token = next(g)[0:2] - if token != '.': - break - tokentype, token = next(g)[0:2] - if tokentype != NAME: - break - parts.append(token) - return (".".join(parts), token) +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + cbrowser = _ClassBrowser(fullmodule, path, fname, tree, inpackage) + cbrowser.visit(ast.parse(source)) + return cbrowser.tree def _main(): diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py index 19ccd62dead8ee..d63627004bbad1 100644 --- a/Lib/test/pyclbr_input.py +++ b/Lib/test/pyclbr_input.py @@ -17,12 +17,7 @@ class C (B): d = 10 - # XXX: This causes test_pyclbr.py to fail, but only because the - # introspection-based is_method() code in the test can't - # distinguish between this and a genuine method function like m(). - # The pyclbr.py module gets this right as it parses the text. - # - #f = f + f = f def m(self): pass diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 869799cfa9a66b..01a3027297eee2 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -150,9 +150,6 @@ def test_easy(self): self.checkModule('difflib', ignore=("Match",)) def test_decorators(self): - # XXX: See comment in pyclbr_input.py for a test that would fail - # if it were not commented out. - # self.checkModule('test.pyclbr_input', ignore=['om']) def test_nested(self): @@ -160,10 +157,10 @@ def test_nested(self): # Set arguments for descriptor creation and _creat_tree call. m, p, f, t, i = 'test', '', 'test.py', {}, None source = dedent("""\ - def f0: + def f0(): def f1(a,b,c): def f2(a=1, b=2, c=3): pass - return f1(a,b,d) + return f1(a,b,d) class c1: pass class C0: "Test class." @@ -178,16 +175,27 @@ def F3(): return 1+1 """) actual = mb._create_tree(m, p, f, source, t, i) - # Create descriptors, linked together, and expected dict. + def _nest_function(ob, func_name, lineno): + newfunc = mb.Function(ob.module, func_name, ob.file, lineno, ob) + ob._addchild(func_name, newfunc) + if isinstance(ob, mb.Class): + ob._addmethod(func_name, lineno) + return newfunc + + def _nest_class(ob, class_name, lineno, super=None): + newclass = mb.Class(ob.module, class_name, super, ob.file, lineno, ob) + ob._addchild(class_name, newclass) + return newclass + f0 = mb.Function(m, 'f0', f, 1) - f1 = mb._nest_function(f0, 'f1', 2) - f2 = mb._nest_function(f1, 'f2', 3) - c1 = mb._nest_class(f0, 'c1', 5) + f1 = _nest_function(f0, 'f1', 2) + f2 = _nest_function(f1, 'f2', 3) + c1 = _nest_class(f0, 'c1', 5) C0 = mb.Class(m, 'C0', None, f, 6) - F1 = mb._nest_function(C0, 'F1', 8) - C1 = mb._nest_class(C0, 'C1', 11) - C2 = mb._nest_class(C1, 'C2', 12) - F3 = mb._nest_function(C2, 'F3', 14) + F1 = _nest_function(C0, 'F1', 8) + C1 = _nest_class(C0, 'C1', 11) + C2 = _nest_class(C1, 'C2', 12) + F3 = _nest_function(C2, 'F3', 14) expected = {'f0':f0, 'C0':C0} def compare(parent1, children1, parent2, children2): diff --git a/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst b/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst new file mode 100644 index 00000000000000..32c4d7fb829f23 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst @@ -0,0 +1,3 @@ +Rewrite :mod:`pyclbr` with the support of abstract syntax trees. Add an +``is_async`` identifier to :mod:`pyclbr`'s ``Function`` objects. Patch by +Batuhan Taskaya From 52fffd6096c759dc7a70a16af5eb503c49e2a9fd Mon Sep 17 00:00:00 2001 From: isidentical Date: Tue, 21 Jan 2020 16:59:47 +0300 Subject: [PATCH 02/10] add some helper functions to idlelib for removed private API --- Lib/idlelib/idle_test/test_browser.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 25d6dc6630364b..cce5b216bf156e 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -60,16 +60,28 @@ def test_close(self): # Nested tree same as in test_pyclbr.py except for supers on C0. C1. mb = pyclbr +def _nest_function(ob, func_name, lineno): + newfunc = mb.Function(ob.module, func_name, ob.file, lineno, ob) + ob._addchild(func_name, newfunc) + if isinstance(ob, mb.Class): + ob._addmethod(func_name, lineno) + return newfunc + +def _nest_class(ob, class_name, lineno, super=None): + newclass = mb.Class(ob.module, class_name, super, ob.file, lineno, ob) + ob._addchild(class_name, newclass) + return newclass + module, fname = 'test', 'test.py' C0 = mb.Class(module, 'C0', ['base'], fname, 1) -F1 = mb._nest_function(C0, 'F1', 3) -C1 = mb._nest_class(C0, 'C1', 6, ['']) -C2 = mb._nest_class(C1, 'C2', 7) -F3 = mb._nest_function(C2, 'F3', 9) +F1 = _nest_function(C0, 'F1', 3) +C1 = _nest_class(C0, 'C1', 6, ['']) +C2 = _nest_class(C1, 'C2', 7) +F3 = _nest_function(C2, 'F3', 9) f0 = mb.Function(module, 'f0', fname, 11) -f1 = mb._nest_function(f0, 'f1', 12) -f2 = mb._nest_function(f1, 'f2', 13) -c1 = mb._nest_class(f0, 'c1', 15) +f1 = _nest_function(f0, 'f1', 12) +f2 = _nest_function(f1, 'f2', 13) +c1 = _nest_class(f0, 'c1', 15) mock_pyclbr_tree = {'C0': C0, 'f0': f0} # Adjust C0.name, C1.name so tests do not depend on order. From f3ff250a79c034db04b466a5ba281d291e77c85a Mon Sep 17 00:00:00 2001 From: isidentical Date: Tue, 21 Jan 2020 17:25:42 +0300 Subject: [PATCH 03/10] apply Pablo's suggestions --- Doc/library/pyclbr.rst | 2 +- Lib/pyclbr.py | 26 +++++++++++-------- .../2020-01-21-16-38-25.bpo-39411.9uHFqT.rst | 5 ++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 15f28c1f52e868..85a68c04fd6c4f 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -96,7 +96,7 @@ statements. They have the following attributes: .. attribute:: Function.parent - For functions that are defined with ``async`` prefix, ``True``. + ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. .. versionadded:: 3.9 diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 86119d83ecfa7d..023c0e590a6d93 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -27,7 +27,7 @@ Instances of Function describe functions with the attributes from _Object, plus the following: - is_async -- if function defined with 'async' prefix + is_async -- if a function is defined with an 'async' prefix Instances of Class describe classes with the attributes from _Object, plus the following: @@ -188,8 +188,7 @@ def visit_ClassDef(self, node): elif len(names := name.split(".")) > 1: # Super class form is module.class: # look in module for class. - module = names[-2] - class_ = names[-1] + *_, module, class_ = names if module in _modules: bases.append(_modules[module].get(class_) or name) else: @@ -206,14 +205,7 @@ def visit_ClassDef(self, node): self.stack.pop() def visit_Assign(self, node): - if ( - not len(node.targets) == 1 - or not len(self.stack) > 0 - or not isinstance(self.stack[-1], Class) - or not isinstance(node.targets[0], ast.Name) - or not isinstance(node.value, ast.Name) - or not isinstance(self.tree.get(node.value.id), Function) - ): + if not self.single_target_function_assign(node): return name = node.targets[0].id @@ -222,6 +214,18 @@ def visit_Assign(self, node): self.stack[-1]._addchild(name, child) self.stack[-1]._addmethod(name, node.lineno) + def single_target_function_assign(self, node): + """Check if given assignment consists from a single target + and single value within a class namespace. Check value for if it + is an already defined function.""" + + return (len(node.targets) == 1 + and len(self.stack) > 0 + and isinstance(self.stack[-1], Class) + and isinstance(node.targets[0], ast.Name) + and isinstance(node.value, ast.Name) + and isinstance(self.tree.get(node.value.id), Function)) + def visit_FunctionDef(self, node, *, is_async=True): parent = self.stack[-1] if self.stack else None function = Function( diff --git a/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst b/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst index 32c4d7fb829f23..2377eef4b9f717 100644 --- a/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst +++ b/Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst @@ -1,3 +1,2 @@ -Rewrite :mod:`pyclbr` with the support of abstract syntax trees. Add an -``is_async`` identifier to :mod:`pyclbr`'s ``Function`` objects. Patch by -Batuhan Taskaya +Add an ``is_async`` identifier to :mod:`pyclbr`'s ``Function`` objects. +Patch by Batuhan Taskaya From 4547e35a38888bc63e0b5f34ae3c829de945eb4d Mon Sep 17 00:00:00 2001 From: isidentical Date: Tue, 21 Jan 2020 17:26:40 +0300 Subject: [PATCH 04/10] get default --- Lib/pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 023c0e590a6d93..a04f4cb70e8aa7 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -190,7 +190,7 @@ def visit_ClassDef(self, node): # look in module for class. *_, module, class_ = names if module in _modules: - bases.append(_modules[module].get(class_) or name) + bases.append(_modules[module].get(class_, name)) else: bases.append(name) From bf12f16481428ab349f9fb14395b5b43546ffd83 Mon Sep 17 00:00:00 2001 From: isidentical Date: Tue, 21 Jan 2020 17:43:25 +0300 Subject: [PATCH 05/10] avoid copying whole Function --- Lib/pyclbr.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index a04f4cb70e8aa7..14a724e0043dc1 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -209,10 +209,14 @@ def visit_Assign(self, node): return name = node.targets[0].id - child = copy.deepcopy(self.tree[node.value.id]) - child.parent = self.stack[-1] - self.stack[-1]._addchild(name, child) - self.stack[-1]._addmethod(name, node.lineno) + value = self.tree[node.value.id] + parent = self.stack[-1] + child = Function( + value.module, name, value.file, node.lineno, parent, value.is_async + ) + child.children = copy.deepcopy(value.children) + parent._addchild(name, child) + parent._addmethod(name, node.lineno) def single_target_function_assign(self, node): """Check if given assignment consists from a single target From 1359246298f1d4c126cd72cc8cc7c0191ee34d09 Mon Sep 17 00:00:00 2001 From: isidentical Date: Wed, 22 Jan 2020 09:38:05 +0300 Subject: [PATCH 06/10] put _nest_function, _nest_class inside pyclbr again, for tests --- Doc/library/pyclbr.rst | 2 +- Lib/idlelib/idle_test/test_browser.py | 26 +++++++------------------- Lib/pyclbr.py | 24 ++++++++++++++++++++---- Lib/test/test_pyclbr.py | 27 ++++++++------------------- 4 files changed, 36 insertions(+), 43 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 85a68c04fd6c4f..cc4eb981fb33df 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -94,7 +94,7 @@ statements. They have the following attributes: .. versionadded:: 3.7 -.. attribute:: Function.parent +.. attribute:: Function.is_async ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index cce5b216bf156e..25d6dc6630364b 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -60,28 +60,16 @@ def test_close(self): # Nested tree same as in test_pyclbr.py except for supers on C0. C1. mb = pyclbr -def _nest_function(ob, func_name, lineno): - newfunc = mb.Function(ob.module, func_name, ob.file, lineno, ob) - ob._addchild(func_name, newfunc) - if isinstance(ob, mb.Class): - ob._addmethod(func_name, lineno) - return newfunc - -def _nest_class(ob, class_name, lineno, super=None): - newclass = mb.Class(ob.module, class_name, super, ob.file, lineno, ob) - ob._addchild(class_name, newclass) - return newclass - module, fname = 'test', 'test.py' C0 = mb.Class(module, 'C0', ['base'], fname, 1) -F1 = _nest_function(C0, 'F1', 3) -C1 = _nest_class(C0, 'C1', 6, ['']) -C2 = _nest_class(C1, 'C2', 7) -F3 = _nest_function(C2, 'F3', 9) +F1 = mb._nest_function(C0, 'F1', 3) +C1 = mb._nest_class(C0, 'C1', 6, ['']) +C2 = mb._nest_class(C1, 'C2', 7) +F3 = mb._nest_function(C2, 'F3', 9) f0 = mb.Function(module, 'f0', fname, 11) -f1 = _nest_function(f0, 'f1', 12) -f2 = _nest_function(f1, 'f2', 13) -c1 = _nest_class(f0, 'c1', 15) +f1 = mb._nest_function(f0, 'f1', 12) +f2 = mb._nest_function(f1, 'f2', 13) +c1 = mb._nest_class(f0, 'c1', 15) mock_pyclbr_tree = {'C0': C0, 'f0': f0} # Adjust C0.name, C1.name so tests do not depend on order. diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 14a724e0043dc1..1ebbd603f5b75d 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -84,6 +84,22 @@ def __init__(self, module, name, super_, file, lineno, parent=None): def _addmethod(self, name, lineno): self.methods[name] = lineno +# This 2 functions are used in these tests +# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py +def _nest_function(ob, func_name, lineno, is_async=False): + "Return a Function after nesting within ob." + newfunc = Function(ob.module, func_name, ob.file, lineno, ob, is_async) + ob._addchild(func_name, newfunc) + if isinstance(ob, Class): + ob._addmethod(func_name, lineno) + return newfunc + +def _nest_class(ob, class_name, lineno, super=None): + "Return a Class after nesting within ob." + newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) + ob._addchild(class_name, newclass) + return newclass + def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -169,7 +185,7 @@ def _readmodule(module, path, inpackage=None): return _create_tree(fullmodule, path, fname, source, tree, inpackage) -class _ClassBrowser(ast.NodeVisitor): +class _ModuleBrowser(ast.NodeVisitor): def __init__(self, module, path, file, tree, inpackage): self.path = path self.tree = tree @@ -281,9 +297,9 @@ def visit_ImportFrom(self, node): def _create_tree(fullmodule, path, fname, source, tree, inpackage): - cbrowser = _ClassBrowser(fullmodule, path, fname, tree, inpackage) - cbrowser.visit(ast.parse(source)) - return cbrowser.tree + mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage) + mbrowser.visit(ast.parse(source)) + return mbrowser.tree def _main(): diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 01a3027297eee2..2c7afa994f3058 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -175,27 +175,16 @@ def F3(): return 1+1 """) actual = mb._create_tree(m, p, f, source, t, i) - def _nest_function(ob, func_name, lineno): - newfunc = mb.Function(ob.module, func_name, ob.file, lineno, ob) - ob._addchild(func_name, newfunc) - if isinstance(ob, mb.Class): - ob._addmethod(func_name, lineno) - return newfunc - - def _nest_class(ob, class_name, lineno, super=None): - newclass = mb.Class(ob.module, class_name, super, ob.file, lineno, ob) - ob._addchild(class_name, newclass) - return newclass - + # Create descriptors, linked together, and expected dict. f0 = mb.Function(m, 'f0', f, 1) - f1 = _nest_function(f0, 'f1', 2) - f2 = _nest_function(f1, 'f2', 3) - c1 = _nest_class(f0, 'c1', 5) + f1 = mb._nest_function(f0, 'f1', 2) + f2 = mb._nest_function(f1, 'f2', 3) + c1 = mb._nest_class(f0, 'c1', 5) C0 = mb.Class(m, 'C0', None, f, 6) - F1 = _nest_function(C0, 'F1', 8) - C1 = _nest_class(C0, 'C1', 11) - C2 = _nest_class(C1, 'C2', 12) - F3 = _nest_function(C2, 'F3', 14) + F1 = mb._nest_function(C0, 'F1', 8) + C1 = mb._nest_class(C0, 'C1', 11) + C2 = mb._nest_class(C1, 'C2', 12) + F3 = mb._nest_function(C2, 'F3', 14) expected = {'f0':f0, 'C0':C0} def compare(parent1, children1, parent2, children2): From d34c9f3b8dce06913525bb63b18a216d3c6fa433 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 19 Oct 2020 19:38:17 +0300 Subject: [PATCH 07/10] 3.10 --- Doc/library/pyclbr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index cc4eb981fb33df..8981213eb73bdd 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -98,7 +98,7 @@ statements. They have the following attributes: ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. - .. versionadded:: 3.9 + .. versionadded:: 3.10 .. _pyclbr-class-objects: From c7b8351d00f97ea20270adf37b6ebefa93e87369 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 20 Oct 2020 15:54:51 +0300 Subject: [PATCH 08/10] Get rid of _addchild / _addmethod methods --- Lib/pyclbr.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 1ebbd603f5b75d..fc456014cb2929 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -60,11 +60,7 @@ def __init__(self, module, name, file, lineno, parent): self.parent = parent self.children = {} if parent is not None: - parent._addchild(name, self) - - def _addchild(self, name, obj): - self.children[name] = obj - + parent.children[name] = self class Function(_Object): "Information about a Python function, including methods." @@ -72,7 +68,7 @@ def __init__(self, module, name, file, lineno, parent=None, is_async=False): super().__init__(module, name, file, lineno, parent) self.is_async = is_async if isinstance(parent, Class): - parent._addmethod(name, lineno) + parent.methods[name] = lineno class Class(_Object): "Information about a Python class." @@ -81,24 +77,15 @@ def __init__(self, module, name, super_, file, lineno, parent=None): self.super = super_ or [] self.methods = {} - def _addmethod(self, name, lineno): - self.methods[name] = lineno - -# This 2 functions are used in these tests +# These 2 functions are used in these tests # Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py def _nest_function(ob, func_name, lineno, is_async=False): "Return a Function after nesting within ob." - newfunc = Function(ob.module, func_name, ob.file, lineno, ob, is_async) - ob._addchild(func_name, newfunc) - if isinstance(ob, Class): - ob._addmethod(func_name, lineno) - return newfunc + return Function(ob.module, func_name, ob.file, lineno, ob, is_async) def _nest_class(ob, class_name, lineno, super=None): "Return a Class after nesting within ob." - newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) - ob._addchild(class_name, newclass) - return newclass + return Class(ob.module, class_name, super, ob.file, lineno, ob) def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -231,8 +218,8 @@ def visit_Assign(self, node): value.module, name, value.file, node.lineno, parent, value.is_async ) child.children = copy.deepcopy(value.children) - parent._addchild(name, child) - parent._addmethod(name, node.lineno) + parent.children[name] = child + parent.methods[name] = node.lineno def single_target_function_assign(self, node): """Check if given assignment consists from a single target From dbf563ad368ba383d5b5d7a158c9146819b538d6 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 10 Nov 2020 13:27:06 +0300 Subject: [PATCH 09/10] Revert support for method aliases --- Lib/pyclbr.py | 26 -------------------------- Lib/test/pyclbr_input.py | 7 ++++++- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index fc456014cb2929..53423f9dc3579a 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -207,32 +207,6 @@ def visit_ClassDef(self, node): self.generic_visit(node) self.stack.pop() - def visit_Assign(self, node): - if not self.single_target_function_assign(node): - return - - name = node.targets[0].id - value = self.tree[node.value.id] - parent = self.stack[-1] - child = Function( - value.module, name, value.file, node.lineno, parent, value.is_async - ) - child.children = copy.deepcopy(value.children) - parent.children[name] = child - parent.methods[name] = node.lineno - - def single_target_function_assign(self, node): - """Check if given assignment consists from a single target - and single value within a class namespace. Check value for if it - is an already defined function.""" - - return (len(node.targets) == 1 - and len(self.stack) > 0 - and isinstance(self.stack[-1], Class) - and isinstance(node.targets[0], ast.Name) - and isinstance(node.value, ast.Name) - and isinstance(self.tree.get(node.value.id), Function)) - def visit_FunctionDef(self, node, *, is_async=True): parent = self.stack[-1] if self.stack else None function = Function( diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py index d63627004bbad1..19ccd62dead8ee 100644 --- a/Lib/test/pyclbr_input.py +++ b/Lib/test/pyclbr_input.py @@ -17,7 +17,12 @@ class C (B): d = 10 - f = f + # XXX: This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). + # The pyclbr.py module gets this right as it parses the text. + # + #f = f def m(self): pass From 0399dcf857026f1fb1b1c98d8136ad9b53e40161 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 10 Nov 2020 18:08:57 +0300 Subject: [PATCH 10/10] set is_async=False by default --- Lib/pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 53423f9dc3579a..f0c8381946c614 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -207,7 +207,7 @@ def visit_ClassDef(self, node): self.generic_visit(node) self.stack.pop() - def visit_FunctionDef(self, node, *, is_async=True): + def visit_FunctionDef(self, node, *, is_async=False): parent = self.stack[-1] if self.stack else None function = Function( self.module, node.name, self.file, node.lineno, parent, is_async