From b615c54de62a6e4a6e1a98375fb195ec3c08b64d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 16 Jan 2020 23:56:25 +0000 Subject: [PATCH 01/29] bpo-17005: Add a topological sort algorithm --- Doc/library/functools.rst | 111 +++++++++++ Doc/whatsnew/3.9.rst | 7 + Lib/functools.py | 185 ++++++++++++++++++ Lib/test/test_functools.py | 140 +++++++++++++ .../2020-01-17-00-00-58.bpo-17005.nTSxsy.rst | 3 + 5 files changed, 446 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index bb7aac42daca78..ef46062d8ef77c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -512,6 +512,117 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 +.. class:: TopologicalSorter(graph=None) + + Provides functionality to topologically sort a graph of hashable nodes. + + A topological order is a linear ordering of the vertices in a graph such that for + every directed edge u -> v from vertex u to vertex v, vertex u comes before vertex + v in the ordering. For instance, the vertices of the graph may represent tasks to + be performed, and the edges may represent constraints that one task must be + performed before another; in this example, a topological ordering is just a valid + sequence for the tasks. A complete topological ordering is possible if and only if + the graph has no directed cycles, that is, if it is a directed acyclic graph. + + If the optional *graph* argument is provided it must be a dictionary representing + a direct acyclic graph where the keys are nodes and the values are iterables of + all predecessors of that node in the graph (the nodes that have edges that point + to the value in the key). Additional nodes can be added to the graph using the + :meth:`~TopologicalSorter.add` method. + + Using the methods provided by this class, a stable topological order of the nodes in the + graph can be derived easily:: + + def stable_topological_order(graph): + ts = TopologicalSorter(graph) + ts.prepare() + while ts.is_active(): + for node in ts.get_ready(): + yield node + ts.done(node) + + This function can be used to implement a simple version of the C3 + linearization algorithm used by Python to calculate the Method Resolution + Order (MRO) of a derived class:: + + >>> class A: pass + >>> class B(A): pass + >>> class C(A): pass + >>> class D(B, C): pass + + >>> D.__mro__ + (__main__.D, __main__.B, __main__.C, __main__.A, object) + + >>> topological_order = tuple(stable_topological_order(graph)) + >>> tuple(reversed(topological_order)) + (__main__.D, '__main__.B, __main__.C, __main__.A, object) + + The class is designed to easily support parallel processing of the nodes as they + become ready. For example:: + + topological_sorter = TopologicalSorter() + + # Add nodes to 'topological_sorter'... + + topological_sorter.prepare() + while topological_sorter.is_active(): + for node in topological_sorter.get_ready(): + # Worker threads or processes take nodes to work on off the + # of 'task_queue' queue. + task_queue.put(node) + + # When the work for a node is done, workers put the node in + # 'finalized_tasks_queue' so we can get more nodes to work on + node = finalized_tasks_queue.get() + + topological_sorter.done(node) + + .. method:: add(node, *predecessors) + + Add a new node and its predecessors to the graph. Both the *node* and + all elements in *predecessors* must be hashable. + + Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. + + .. method:: prepare() + + Mark the graph as finished and check for cycles in the graph. If any cycle is detected, + :exc:`CycleError` will be raised, but :meth:`~TopologicalSorter.get_ready` can still be + used to obtain as many nodes as possible until cycles block more progress. After a call + to this function, the graph cannot be modified and therefore no more nodes can be added + using :meth:`~TopologicalSorter.add`. + + .. method:: is_active() + + Returns ``True`` if more progress can be made and ``False`` otherwise. Progress can be + made if cycles do not block the resolution and either there are still nodes ready that haven't + yet been returned by :meth:`TopologicalSorter.get_ready` or the number of nodes marked + :meth:`TopologicalSorter.done` is less than the number that have been returned by + :meth:`TopologicalSorter.get_ready`. + + Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` previously. + + .. method:: done(node) + + Marks a node returned by :meth:`TopologicalSorter.get_ready` as processed, unblocking any + successor of *node* for being returned in the future by a call to :meth:`TopologicalSorter.get_ready`. + + Raises :exc:`ValueError` if *node* has already been marked as processed by a previous call to this + method or if *node* was not added to the graph by using :meth:`TopologicalSorter.add` or if called without + calling :meth:`~TopologicalSorter.prepare` previously. + + .. method:: get_ready() + + Returns a ``tuple`` with all the nodes that are ready. Initially it returns all nodes with no + predecessors and once those are marked as processed by calling :meth:`TopologicalSorter.done`, + further calls will return all new nodes that have all their predecessors already processed until + no more progress can be made. + + Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` previously. + + .. versionadded:: 3.9 + + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) Update a *wrapper* function to look like the *wrapped* function. The optional diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index f40685c932793f..73c6232033e7d7 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -166,6 +166,13 @@ ftplib if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) +functools +--------- + +Add the :class:`functools.TopologicalSorter` class to offer functionality to perform +topological sorting of graphs. (Contributed by Pablo Galindo and Tim Peters in +:issue:`17005`.) + gc -- diff --git a/Lib/functools.py b/Lib/functools.py index 2c01b2e59524bf..9deb754b6cc312 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -192,6 +192,191 @@ def total_ordering(cls): setattr(cls, opname, opfunc) return cls +################################################################################ +### topological sort +################################################################################ + +_NODE_OUT = -1 +_NODE_DONE = -2 + +class _NodeInfo: + __slots__ = 'node', 'npredecessors', 'successors' + + def __init__(self, node): + # The node this class is augmenting. + self.node = node + + # Number of predecessors, generally >= 0. When this value falls to 0, + # and is returned by get_ready(), this is set to _NODE_OUT and when the + # node is marked done by a call to done(), set to _NODE_DONE. + self.npredecessors = 0 + + # List of successor nodes. The list can contain duplicated elements as + # long as they're all reflected in the successor's npredecessors attribute). + self.successors = [] + +class CycleError(ValueError): + pass + +class TopologicalSorter: + """Provides functionality to topologically sort a graph of hashable nodes""" + + def __init__(self, graph=None): + self.node2info = {} + self.ready_nodes = None + self.npassedout = 0 + self.nfinished = 0 + + if graph is not None: + for node, predecessors in graph.items(): + self.add(node, *predecessors) + + def _get_nodeinfo(self, node): + if (result := self.node2info.get(node)) is None: + self.node2info[node] = result = _NodeInfo(node) + return result + + def add(self, node, *predecessors): + """Add a new node and its predecessors to the graph. + + Both the *node* and all elements in *predecessors* must be hashable. + + Raises ValueError if called after "prepare". + """ + if self.ready_nodes is not None: + raise ValueError("Nodes cannot be added after a call to prepare()") + + # Create the node -> predecessor edges + nodeinfo = self._get_nodeinfo(node) + nodeinfo.npredecessors += len(predecessors) + + # Create the predecessor -> node edges + for pred in predecessors: + pred_info = self._get_nodeinfo(pred) + pred_info.successors.append(node) + + def prepare(self): + """Mark the graph as finished and check for cycles in the graph. + + If any cycle is detected, "CycleError" will be raised, but "get_ready" can still be + used to obtain as many nodes as possible until cycles block more progress. After a call + to this function, the graph cannot be modified and therefore no more nodes can be added + using "add". + """ + if self.ready_nodes is not None: + raise ValueError("cannot prepare() more than once") + + self.ready_nodes = [i.node for i in self.node2info.values() + if i.npredecessors == 0] + # readytodo is set before we look for cycles on purpose: + # if the user wants to catch the CycleError, that's fine, + # they can continue using the instance to grab as many + # nodes as possible before cycles block more progress + cycle = self._find_cycle() + if cycle: + raise CycleError(f"nodes are in a cycle", cycle) + + def get_ready(self): + """Return a tuple of all the nodes that are ready. + + Initially it returns all nodes with no predecessors and once those are marked as processed by + calling "done", further calls will return all new nodes that have all their predecessors already + processed until no more progress can be made. + + Raises ValueError if called without calling "prepare" previously. + """ + if self.ready_nodes is None: + raise ValueError("prepare() must be called first") + + # Get the nodes that are ready and mark them + result = tuple(self.ready_nodes) + n2i = self.node2info + for node in result: + n2i[node].npredecessors = _NODE_OUT + + # Clean the list of nodes that are ready and update + # the counter of nodes that we have returned. + self.ready_nodes.clear() + self.npassedout += len(result) + + return result + + def is_active(self): + """Return True if more progress can be made and ``False`` otherwise. + + Progress can be made if cycles do not block the resolution and either there are still nodes ready + that haven't yet been returned by "get_ready" or the number of nodes marked "done" is less than the + number that have been returned by "get_ready". + + Raises ValueError if called without calling "prepare" previously. + """ + if self.ready_nodes is None: + raise ValueError("prepare() must be called first") + return self.nfinished < self.npassedout or bool(self.ready_nodes) + + + def done(self, node): + """Marks a nodes returned by "get_ready" as processed. + + This method unblocks any successor of *node* for being returned in the future by a call to "get_ready" + + Raises :exec:`ValueError` if *node* has already been marked as processed by a previous call to this + method or if *node* was not added to the graph by using "add" or if called without calling "prepare" + previously. + """ + + if self.ready_nodes is None: + raise ValueError("prepare() must be called first") + + n2i = self.node2info + + # Check if we know about this node (it was added previously using add() + if (nodeinfo := n2i.get(node)) is None: + raise ValueError(f"node {node!r} was not added using add()") + + # If the node has not being returned (marked as ready) previously, inform the user. + stat = nodeinfo.npredecessors + if stat != _NODE_OUT: + if stat >= 0: + raise ValueError(f"node {node!r} was not passed out (still not ready)") + elif stat == _NODE_DONE: + raise ValueError(f"node {node!r} was already marked done") + else: + raise ValueError(f"node {node!r}: unknown status {stat}") + + # Mark the node as processed + nodeinfo.npredecessors = _NODE_DONE + + # Go to all the successors and reduce the number of predecessors, collecting all the ones + # that are ready to be returned in the next get_ready() call. + for successor in nodeinfo.successors: + successor_info = n2i[successor] + successor_info.npredecessors -= 1 + if successor_info.npredecessors == 0: + self.ready_nodes.append(successor) + self.nfinished += 1 + + def _find_cycle(self): + todo = set(node for node in self.node2info) + for info in self.node2info.values(): + todo |= set(info.successors) + + while todo: + node = todo.pop() + stack = [node] + while stack: + top = stack[-1] + for node in self.node2info[top].successors: + if node in stack: + return stack[stack.index(node):] + [node] + if node in todo: + stack.append(node) + todo.remove(node) + break + else: + node = stack.pop() + return None + ################################################################################ ### cmp_to_key() function converter diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index a97ca398e77a33..0f8023e1f4e514 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1158,6 +1158,146 @@ def __eq__(self, other): return self.value == other.value +class TestTopologicalSort(unittest.TestCase): + + def _test_graph(self, graph, expected): + + def static_order(ts): + ts.prepare() + while ts.is_active(): + nodes = ts.get_ready() + for node in nodes: + ts.done(node) + yield nodes + + ts = functools.TopologicalSorter(graph) + self.assertEqual(list(static_order(ts)), expected) + + def _assert_cycle(self, graph, cycle): + ts = functools.TopologicalSorter() + for node, dependson in graph.items(): + ts.add(node, *dependson) + try: + ts.prepare() + except functools.CycleError as e: + msg, seq = e.args + self.assertIn(' '.join(map(str, cycle)), + ' '.join(map(str, seq * 2))) + else: + raise + + def test_simple_cases(self): + self._test_graph( + {2: {11}, + 9: {11, 8}, + 10: {11, 3}, + 11: {7, 5}, + 8: {7, 3}}, + [(3, 5, 7), (11, 8), (2, 10, 9)] + ) + + self._test_graph({1: {}}, [(1,)]) + + def test_no_dependencies(self): + self._test_graph( + {1: {2}, + 3: {4}, + 5: {6}}, + [(2, 4, 6), (1, 3, 5)] + ) + + self._test_graph( + {1: set(), + 3: set(), + 5: set()}, + [(1, 3, 5)] + ) + + def test_empty(self): + self._test_graph({}, []) + + def test_cycle(self): + # Self cycle + self._assert_cycle({1: {1}}, [1, 1]) + # Simple cycle + self._assert_cycle({1: {2}, 2: {1}}, [1, 2, 1]) + # Indirect cycle + self._assert_cycle({1: {2}, 2: {3}, 3: {1}}, [1, 3, 2, 1]) + # not all elements involved in a cycle + self._assert_cycle({1: {2}, 2: {3}, 3: {1}, 5: {4}, 4: {6}}, [1, 3, 2, 1]) + + def test_calls_before_preapare(self): + ts = functools.TopologicalSorter() + + with self.assertRaisesRegex(ValueError, r"prepare\(\) must be called first"): + ts.get_ready() + with self.assertRaisesRegex(ValueError, r"prepare\(\) must be called first"): + ts.done(3) + with self.assertRaisesRegex(ValueError, r"prepare\(\) must be called first"): + ts.is_active() + + def test_prepare_multiple_times(self): + ts = functools.TopologicalSorter() + ts.prepare() + with self.assertRaisesRegex(ValueError, r"cannot prepare\(\) more than once"): + ts.prepare() + + def test_invalid_nodes_in_done(self): + ts = functools.TopologicalSorter() + ts.add(1, 2, 3, 4) + ts.add(2, 3, 4) + ts.prepare() + ts.get_ready() + + with self.assertRaisesRegex(ValueError, "node 2 was not passed out"): + ts.done(2) + with self.assertRaisesRegex(ValueError, r"node 24 was not added using add\(\)"): + ts.done(24) + + def test_done(self): + ts = functools.TopologicalSorter() + ts.add(1, 2, 3, 4) + ts.add(2, 3) + ts.prepare() + + self.assertEqual(ts.get_ready(), (3, 4)) + # If we don't mark anything as done, get_ready() returns nothing + self.assertEqual(ts.get_ready(), ()) + ts.done(3) + # Now 2 becomes available as 3 is done + self.assertEqual(ts.get_ready(), (2,)) + self.assertEqual(ts.get_ready(), ()) + ts.done(4) + ts.done(2) + # Only 1 is missing + self.assertEqual(ts.get_ready(), (1,)) + self.assertEqual(ts.get_ready(), ()) + ts.done(1) + self.assertEqual(ts.get_ready(), ()) + self.assertFalse(ts.is_active()) + + def test_is_active(self): + ts = functools.TopologicalSorter() + ts.add(1, 2) + ts.prepare() + + self.assertTrue(ts.is_active()) + self.assertEqual(ts.get_ready(), (2,)) + self.assertTrue(ts.is_active()) + ts.done(2) + self.assertTrue(ts.is_active()) + self.assertEqual(ts.get_ready(), (1,)) + self.assertTrue(ts.is_active()) + ts.done(1) + self.assertFalse(ts.is_active()) + + def test_not_hashable_nodes(self): + ts = functools.TopologicalSorter() + self.assertRaises(TypeError, ts.add, dict(), 1) + self.assertRaises(TypeError, ts.add, 1, dict()) + self.assertRaises(TypeError, ts.add, dict(), dict()) + + class TestLRU: def test_lru(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst new file mode 100644 index 00000000000000..ff573d918ccb4f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst @@ -0,0 +1,3 @@ +Add :class:`functools.TopologicalSorter` to the :mod:`functools` module to +offers functionality to perform topological sorting of graphs. Patch by +Pablo Galindo and Tim Peters. From e9114593efc4705ba6855847c1524e597559ceaa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 01:59:44 +0000 Subject: [PATCH 02/29] Adress first round of Tim feedback and adjust docstring lenght --- Doc/library/functools.rst | 7 ++++--- Lib/functools.py | 36 ++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index ef46062d8ef77c..685690d325c2c7 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -568,7 +568,7 @@ The :mod:`functools` module defines the following functions: while topological_sorter.is_active(): for node in topological_sorter.get_ready(): # Worker threads or processes take nodes to work on off the - # of 'task_queue' queue. + # 'task_queue' queue. task_queue.put(node) # When the work for a node is done, workers put the node in @@ -608,8 +608,9 @@ The :mod:`functools` module defines the following functions: successor of *node* for being returned in the future by a call to :meth:`TopologicalSorter.get_ready`. Raises :exc:`ValueError` if *node* has already been marked as processed by a previous call to this - method or if *node* was not added to the graph by using :meth:`TopologicalSorter.add` or if called without - calling :meth:`~TopologicalSorter.prepare` previously. + method or if *node* was not added to the graph by using :meth:`TopologicalSorter.add`, if called without + calling :meth:`~TopologicalSorter.prepare` or if node has not yet been returned by + :meth:`~TopologicalSorter.get_ready`. .. method:: get_ready() diff --git a/Lib/functools.py b/Lib/functools.py index 9deb754b6cc312..c310c4612180ce 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -258,17 +258,17 @@ def add(self, node, *predecessors): def prepare(self): """Mark the graph as finished and check for cycles in the graph. - If any cycle is detected, "CycleError" will be raised, but "get_ready" can still be - used to obtain as many nodes as possible until cycles block more progress. After a call - to this function, the graph cannot be modified and therefore no more nodes can be added - using "add". + If any cycle is detected, "CycleError" will be raised, but "get_ready" can + still be used to obtain as many nodes as possible until cycles block more + progress. After a call to this function, the graph cannot be modified and + therefore no more nodes can be added using "add". """ if self.ready_nodes is not None: raise ValueError("cannot prepare() more than once") self.ready_nodes = [i.node for i in self.node2info.values() if i.npredecessors == 0] - # readytodo is set before we look for cycles on purpose: + # ready_nodes is set before we look for cycles on purpose: # if the user wants to catch the CycleError, that's fine, # they can continue using the instance to grab as many # nodes as possible before cycles block more progress @@ -279,9 +279,10 @@ def prepare(self): def get_ready(self): """Return a tuple of all the nodes that are ready. - Initially it returns all nodes with no predecessors and once those are marked as processed by - calling "done", further calls will return all new nodes that have all their predecessors already - processed until no more progress can be made. + Initially it returns all nodes with no predecessors and once those are marked + as processed by calling "done", further calls will return all new nodes that + have all their predecessors already processed until no more progress can be + made. Raises ValueError if called without calling "prepare" previously. """ @@ -304,9 +305,10 @@ def get_ready(self): def is_active(self): """Return True if more progress can be made and ``False`` otherwise. - Progress can be made if cycles do not block the resolution and either there are still nodes ready - that haven't yet been returned by "get_ready" or the number of nodes marked "done" is less than the - number that have been returned by "get_ready". + Progress can be made if cycles do not block the resolution and either there + are still nodes ready that haven't yet been returned by "get_ready" or the + number of nodes marked "done" is less than the number that have been returned + by "get_ready". Raises ValueError if called without calling "prepare" previously. """ @@ -318,11 +320,13 @@ def is_active(self): def done(self, node): """Marks a nodes returned by "get_ready" as processed. - This method unblocks any successor of *node* for being returned in the future by a call to "get_ready" + This method unblocks any successor of *node* for being returned in the future + by a call to "get_ready" - Raises :exec:`ValueError` if *node* has already been marked as processed by a previous call to this - method or if *node* was not added to the graph by using "add" or if called without calling "prepare" - previously. + Raises :exec:`ValueError` if *node* has already been marked as processed by a + previous call to this method, if *node* was not added to the graph by using + "add" or if called without calling "prepare" previously or if node has not + yet been returned by "get_ready". """ if self.ready_nodes is None: @@ -342,7 +346,7 @@ def done(self, node): elif stat == _NODE_DONE: raise ValueError(f"node {node!r} was already marked done") else: - raise ValueError(f"node {node!r}: unknown status {stat}") + assert False, f"node {node!r}: unknown status {stat}" # Mark the node as processed nodeinfo.npredecessors = _NODE_DONE From 493f6a560e7f682c9f516c799a6d08d2712a3b7a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 02:35:12 +0000 Subject: [PATCH 03/29] Add a set to avoid O(n2) behaviour in the check for cycles --- Lib/functools.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index c310c4612180ce..5b90e2d265a548 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -361,24 +361,27 @@ def done(self, node): self.nfinished += 1 def _find_cycle(self): - todo = set(node for node in self.node2info) - for info in self.node2info.values(): - todo |= set(info.successors) + todo = set(self.node2info) while todo: node = todo.pop() stack = [node] + # This set helps avoiding cuadratic behaviour when checking + # if a node is in the stack. + in_stack = {node} while stack: top = stack[-1] for node in self.node2info[top].successors: - if node in stack: + if node in in_stack: return stack[stack.index(node):] + [node] if node in todo: + in_stack.add(node) stack.append(node) todo.remove(node) break else: node = stack.pop() + in_stack.discard(node) return None From cac6cf29ff2322d23d7ebdbc93f44621e2dbedc9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 03:28:51 +0000 Subject: [PATCH 04/29] Use Tim's cycle-search algorithm --- Lib/functools.py | 50 +++++++++++++++++++++++--------------- Lib/test/test_functools.py | 3 +++ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 5b90e2d265a548..11325bc1567ac9 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -361,30 +361,40 @@ def done(self, node): self.nfinished += 1 def _find_cycle(self): - todo = set(self.node2info) - - while todo: - node = todo.pop() - stack = [node] - # This set helps avoiding cuadratic behaviour when checking - # if a node is in the stack. - in_stack = {node} - while stack: - top = stack[-1] - for node in self.node2info[top].successors: - if node in in_stack: - return stack[stack.index(node):] + [node] - if node in todo: - in_stack.add(node) - stack.append(node) - todo.remove(node) + n2i = self.node2info + stack = [] + itstack = [] + seen = set() + node2stacki = {} + + for node in n2i: + if node in seen: + continue + + while True: + # If we have seen already the node and is in the + # current stack we have found a cycle. + if node in seen and node in node2stacki: + return stack[node2stacki[node]:] + [node] + + seen.add(node) + itstack.append(iter(n2i[node].successors).__next__) + node2stacki[node] = len(stack) + stack.append(node) + + # Backtrack to the topmost stack entry with + # at least another successor. + while stack: + try: + node = itstack[-1]() break + except StopIteration: + del node2stacki[stack.pop()] + itstack.pop() else: - node = stack.pop() - in_stack.discard(node) + break return None - ################################################################################ ### cmp_to_key() function converter ################################################################################ diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 0f8023e1f4e514..50fb728042e4ba 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1225,6 +1225,9 @@ def test_cycle(self): self._assert_cycle({1: {2}, 2: {3}, 3: {1}}, [1, 3, 2, 1]) # not all elements involved in a cycle self._assert_cycle({1: {2}, 2: {3}, 3: {1}, 5: {4}, 4: {6}}, [1, 3, 2, 1]) + # Multiple cycles + self._assert_cycle({1: {2}, 2: {1}, 3: {4}, 4: {5}, 6: {7}, 7: {6}}, + [1, 2, 1]) def test_calls_before_preapare(self): ts = functools.TopologicalSorter() From e1255dd3724d949b58c6d45a14e45e5048f5291c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 03:37:38 +0000 Subject: [PATCH 05/29] Add another test for the cycle-find algorithm --- Lib/test/test_functools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 50fb728042e4ba..81e4d9fdaa8516 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1228,6 +1228,8 @@ def test_cycle(self): # Multiple cycles self._assert_cycle({1: {2}, 2: {1}, 3: {4}, 4: {5}, 6: {7}, 7: {6}}, [1, 2, 1]) + # Cycle in the middle of the graph + self._assert_cycle({1: {2}, 2: {3}, 3: {2, 4}, 4: {5}}, [3, 2]) def test_calls_before_preapare(self): ts = functools.TopologicalSorter() From 301a459488fb286d549e22ec104b06a3433e0f94 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 04:00:14 +0000 Subject: [PATCH 06/29] Uncouple if confition and restore original code --- Lib/functools.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 11325bc1567ac9..47506669b2f607 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -372,15 +372,17 @@ def _find_cycle(self): continue while True: - # If we have seen already the node and is in the - # current stack we have found a cycle. - if node in seen and node in node2stacki: - return stack[node2stacki[node]:] + [node] - - seen.add(node) - itstack.append(iter(n2i[node].successors).__next__) - node2stacki[node] = len(stack) - stack.append(node) + if node in seen: + # If we have seen already the node and is in the + # current stack we have found a cycle. + if node in node2stacki: + return stack[node2stacki[node]:] + [node] + # else go on to get next successor + else: + seen.add(node) + itstack.append(iter(n2i[node].successors).__next__) + node2stacki[node] = len(stack) + stack.append(node) # Backtrack to the topmost stack entry with # at least another successor. From 6f33c71dec7f7a9bdfc0a2bc45be4d3ec17e5470 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 04:04:22 +0000 Subject: [PATCH 07/29] Add __bool__ as alias of is_active --- Lib/functools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/functools.py b/Lib/functools.py index 47506669b2f607..68b524b35a4583 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -316,6 +316,8 @@ def is_active(self): raise ValueError("prepare() must be called first") return self.nfinished < self.npassedout or bool(self.ready_nodes) + def __bool__(self): + return self.is_active() def done(self, node): """Marks a nodes returned by "get_ready" as processed. From 1efe994d521ed5dc1e8c05768a18deee380d3567 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 11:40:10 +0000 Subject: [PATCH 08/29] Improve docs and allow multiple nodes in done() --- Doc/library/functools.rst | 61 +++++++++++---- Doc/whatsnew/3.9.rst | 4 +- Lib/functools.py | 76 +++++++++++-------- .../2020-01-17-00-00-58.bpo-17005.nTSxsy.rst | 2 +- 4 files changed, 93 insertions(+), 50 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 685690d325c2c7..f3a38ab1a9bc94 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -530,13 +530,25 @@ The :mod:`functools` module defines the following functions: to the value in the key). Additional nodes can be added to the graph using the :meth:`~TopologicalSorter.add` method. - Using the methods provided by this class, a stable topological order of the nodes in the - graph can be derived easily:: + Normally, the steps required to perform the sorting of a given graph are as follows: + + * Create an instance of the :class:`TopologicalSorter` with an optional + initial graph. + * Add additional nodes to the graph. + * Call "finalize()" on the graph. + * While :meth:`~TopologicalSorter.is_active` is ``True``: + * Iterate over the nodes returned by :meth:`~TopologicalSorter.get_ready` + and process them. Call :meth:`~TopologicalSorter.done` on each node as + it finishes processing. + + + Using the methods provided by this class, a stable topological order of the nodes + in the graph can be derived easily:: def stable_topological_order(graph): ts = TopologicalSorter(graph) ts.prepare() - while ts.is_active(): + while ts: for node in ts.get_ready(): yield node ts.done(node) @@ -582,6 +594,14 @@ The :mod:`functools` module defines the following functions: Add a new node and its predecessors to the graph. Both the *node* and all elements in *predecessors* must be hashable. + If called multiple times with the same node argument, the set of dependencies + will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not provided) + as well as provide a dependency twice. If a node that has not been provided before + is included among *predecessors* it will be automatically added to the graph with + no predecessors of its own. + Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. .. method:: prepare() @@ -595,22 +615,35 @@ The :mod:`functools` module defines the following functions: .. method:: is_active() Returns ``True`` if more progress can be made and ``False`` otherwise. Progress can be - made if cycles do not block the resolution and either there are still nodes ready that haven't - yet been returned by :meth:`TopologicalSorter.get_ready` or the number of nodes marked - :meth:`TopologicalSorter.done` is less than the number that have been returned by + made if cycles do not block the resolution and either there are still nodes ready that + haven't yet been returned by :meth:`TopologicalSorter.get_ready` or the number of nodes + marked :meth:`TopologicalSorter.done` is less than the number that have been returned by :meth:`TopologicalSorter.get_ready`. - Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` previously. + The :meth:`~TopologicalSorter.__bool__` method of this class defers to this function, so + instead of:: + + if ts.is_active(): + ... + + if possible to simply do:: + + if ts: + ... + + Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` + previously. - .. method:: done(node) + .. method:: done(*nodes) - Marks a node returned by :meth:`TopologicalSorter.get_ready` as processed, unblocking any - successor of *node* for being returned in the future by a call to :meth:`TopologicalSorter.get_ready`. + Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as processed, + unblocking any successor of each node in *nodes* for being returned in the future by a + call to :meth:`TopologicalSorter.get_ready`. - Raises :exc:`ValueError` if *node* has already been marked as processed by a previous call to this - method or if *node* was not added to the graph by using :meth:`TopologicalSorter.add`, if called without - calling :meth:`~TopologicalSorter.prepare` or if node has not yet been returned by - :meth:`~TopologicalSorter.get_ready`. + Raises :exc:`ValueError` if any node in *nodes* has already been marked as processed by a + previous call to this method or if a node was not added to the graph by using + :meth:`TopologicalSorter.add`, if called without calling :meth:`~TopologicalSorter.prepare` + or if node has not yet been returned by :meth:`~TopologicalSorter.get_ready`. .. method:: get_ready() diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 73c6232033e7d7..1adb3b70025f6b 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -170,8 +170,8 @@ functools --------- Add the :class:`functools.TopologicalSorter` class to offer functionality to perform -topological sorting of graphs. (Contributed by Pablo Galindo and Tim Peters in -:issue:`17005`.) +topological sorting of graphs. (Contributed by Pablo Galindo, Tim Peters and Larry +Hastings in :issue:`17005`.) gc -- diff --git a/Lib/functools.py b/Lib/functools.py index 68b524b35a4583..dc37de74ec611a 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -241,6 +241,14 @@ def add(self, node, *predecessors): Both the *node* and all elements in *predecessors* must be hashable. + If called multiple times with the same node argument, the set of dependencies + will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not provided) + as well as provide a dependency twice. If a node that has not been provided before + is included among *predecessors* it will be automatically added to the graph with + no predecessors of its own. + Raises ValueError if called after "prepare". """ if self.ready_nodes is not None: @@ -319,16 +327,16 @@ def is_active(self): def __bool__(self): return self.is_active() - def done(self, node): - """Marks a nodes returned by "get_ready" as processed. + def done(self, *nodes): + """Marks a set of nodes returned by "get_ready" as processed. - This method unblocks any successor of *node* for being returned in the future - by a call to "get_ready" + This method unblocks any successor of each node in *nodes* for being returned + in the future by a a call to "get_ready" - Raises :exec:`ValueError` if *node* has already been marked as processed by a - previous call to this method, if *node* was not added to the graph by using - "add" or if called without calling "prepare" previously or if node has not - yet been returned by "get_ready". + Raises :exec:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method, if a node was not added to the + graph by using "add" or if called without calling "prepare" previously or if + node has not yet been returned by "get_ready". """ if self.ready_nodes is None: @@ -336,31 +344,33 @@ def done(self, node): n2i = self.node2info - # Check if we know about this node (it was added previously using add() - if (nodeinfo := n2i.get(node)) is None: - raise ValueError(f"node {node!r} was not added using add()") - - # If the node has not being returned (marked as ready) previously, inform the user. - stat = nodeinfo.npredecessors - if stat != _NODE_OUT: - if stat >= 0: - raise ValueError(f"node {node!r} was not passed out (still not ready)") - elif stat == _NODE_DONE: - raise ValueError(f"node {node!r} was already marked done") - else: - assert False, f"node {node!r}: unknown status {stat}" - - # Mark the node as processed - nodeinfo.npredecessors = _NODE_DONE - - # Go to all the successors and reduce the number of predecessors, collecting all the ones - # that are ready to be returned in the next get_ready() call. - for successor in nodeinfo.successors: - successor_info = n2i[successor] - successor_info.npredecessors -= 1 - if successor_info.npredecessors == 0: - self.ready_nodes.append(successor) - self.nfinished += 1 + for node in nodes: + + # Check if we know about this node (it was added previously using add() + if (nodeinfo := n2i.get(node)) is None: + raise ValueError(f"node {node!r} was not added using add()") + + # If the node has not being returned (marked as ready) previously, inform the user. + stat = nodeinfo.npredecessors + if stat != _NODE_OUT: + if stat >= 0: + raise ValueError(f"node {node!r} was not passed out (still not ready)") + elif stat == _NODE_DONE: + raise ValueError(f"node {node!r} was already marked done") + else: + assert False, f"node {node!r}: unknown status {stat}" + + # Mark the node as processed + nodeinfo.npredecessors = _NODE_DONE + + # Go to all the successors and reduce the number of predecessors, collecting all the ones + # that are ready to be returned in the next get_ready() call. + for successor in nodeinfo.successors: + successor_info = n2i[successor] + successor_info.npredecessors -= 1 + if successor_info.npredecessors == 0: + self.ready_nodes.append(successor) + self.nfinished += 1 def _find_cycle(self): n2i = self.node2info diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst index ff573d918ccb4f..e5336437754af4 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-17-00-00-58.bpo-17005.nTSxsy.rst @@ -1,3 +1,3 @@ Add :class:`functools.TopologicalSorter` to the :mod:`functools` module to offers functionality to perform topological sorting of graphs. Patch by -Pablo Galindo and Tim Peters. +Pablo Galindo, Tim Peters and Larry Hastings. From aa56145a040b2d3fea279124eca85021bfa60b6b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 21:35:53 +0000 Subject: [PATCH 09/29] Add 'static_order' method, docs and tests and some cleanups --- Doc/library/functools.rst | 104 +++++++++++++++++++++---------------- Lib/functools.py | 64 ++++++++++++++--------- Lib/test/test_functools.py | 12 +++-- 3 files changed, 107 insertions(+), 73 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index f3a38ab1a9bc94..5470aef5dfe786 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -530,7 +530,8 @@ The :mod:`functools` module defines the following functions: to the value in the key). Additional nodes can be added to the graph using the :meth:`~TopologicalSorter.add` method. - Normally, the steps required to perform the sorting of a given graph are as follows: + In the general case, the steps required to perform the sorting of a given graph + are as follows: * Create an instance of the :class:`TopologicalSorter` with an optional initial graph. @@ -541,21 +542,11 @@ The :mod:`functools` module defines the following functions: and process them. Call :meth:`~TopologicalSorter.done` on each node as it finishes processing. - - Using the methods provided by this class, a stable topological order of the nodes - in the graph can be derived easily:: - - def stable_topological_order(graph): - ts = TopologicalSorter(graph) - ts.prepare() - while ts: - for node in ts.get_ready(): - yield node - ts.done(node) - - This function can be used to implement a simple version of the C3 - linearization algorithm used by Python to calculate the Method Resolution - Order (MRO) of a derived class:: + In case that just an inmediate sorting of the nodes in the graph is required and + no paralellism is involved, the convencience method :meth:`TopologicalSorter.stable_oder` + can be used directly. For example, using this method is possible to implement a simple + version of the C3 linearization algorithm used by Python to calculate the Method + Resolution Order (MRO) of a derived class:: >>> class A: pass >>> class B(A): pass @@ -565,12 +556,13 @@ The :mod:`functools` module defines the following functions: >>> D.__mro__ (__main__.D, __main__.B, __main__.C, __main__.A, object) - >>> topological_order = tuple(stable_topological_order(graph)) + >>> ts = TopologicalSorter(graph) + >>> topological_order = tuple(ts.stable_order()) >>> tuple(reversed(topological_order)) (__main__.D, '__main__.B, __main__.C, __main__.A, object) The class is designed to easily support parallel processing of the nodes as they - become ready. For example:: + become ready. For instance:: topological_sorter = TopologicalSorter() @@ -597,31 +589,33 @@ The :mod:`functools` module defines the following functions: If called multiple times with the same node argument, the set of dependencies will be the union of all dependencies passed in. - It is possible to add a node with no dependencies (*predecessors* is not provided) - as well as provide a dependency twice. If a node that has not been provided before - is included among *predecessors* it will be automatically added to the graph with - no predecessors of its own. + It is possible to add a node with no dependencies (*predecessors* is not + provided) as well as provide a dependency twice. If a node that has not been + provided before is included among *predecessors* it will be automatically added + to the graph with no predecessors of its own. Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. .. method:: prepare() - Mark the graph as finished and check for cycles in the graph. If any cycle is detected, - :exc:`CycleError` will be raised, but :meth:`~TopologicalSorter.get_ready` can still be - used to obtain as many nodes as possible until cycles block more progress. After a call - to this function, the graph cannot be modified and therefore no more nodes can be added - using :meth:`~TopologicalSorter.add`. + Mark the graph as finished and check for cycles in the graph. If any cycle is + detected, :exc:`CycleError` will be raised, but + :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many nodes + as possible until cycles block more progress. After a call to this function, + the graph cannot be modified and therefore no more nodes can be added using + :meth:`~TopologicalSorter.add`. .. method:: is_active() - Returns ``True`` if more progress can be made and ``False`` otherwise. Progress can be - made if cycles do not block the resolution and either there are still nodes ready that - haven't yet been returned by :meth:`TopologicalSorter.get_ready` or the number of nodes - marked :meth:`TopologicalSorter.done` is less than the number that have been returned by - :meth:`TopologicalSorter.get_ready`. + Returns ``True`` if more progress can be made and ``False`` otherwise. Progress + can be made if cycles do not block the resolution and either there are still + nodes ready that haven't yet been returned by + :meth:`TopologicalSorter.get_ready` or the number of nodes marked + :meth:`TopologicalSorter.done` is less than the number that have been returned + by :meth:`TopologicalSorter.get_ready`. - The :meth:`~TopologicalSorter.__bool__` method of this class defers to this function, so - instead of:: + The :meth:`~TopologicalSorter.__bool__` method of this class defers to this + function, so instead of:: if ts.is_active(): ... @@ -636,23 +630,41 @@ The :mod:`functools` module defines the following functions: .. method:: done(*nodes) - Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as processed, - unblocking any successor of each node in *nodes* for being returned in the future by a - call to :meth:`TopologicalSorter.get_ready`. + Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as + processed, unblocking any successor of each node in *nodes* for being returned + in the future by a call to :meth:`TopologicalSorter.get_ready`. - Raises :exc:`ValueError` if any node in *nodes* has already been marked as processed by a - previous call to this method or if a node was not added to the graph by using - :meth:`TopologicalSorter.add`, if called without calling :meth:`~TopologicalSorter.prepare` - or if node has not yet been returned by :meth:`~TopologicalSorter.get_ready`. + Raises :exc:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method or if a node was not added to the + graph by using :meth:`TopologicalSorter.add`, if called without calling + :meth:`~TopologicalSorter.prepare` or if node has not yet been returned by + :meth:`~TopologicalSorter.get_ready`. .. method:: get_ready() - Returns a ``tuple`` with all the nodes that are ready. Initially it returns all nodes with no - predecessors and once those are marked as processed by calling :meth:`TopologicalSorter.done`, - further calls will return all new nodes that have all their predecessors already processed until - no more progress can be made. + Returns a ``tuple`` with all the nodes that are ready. Initially it returns all + nodes with no predecessors and once those are marked as processed by calling + :meth:`TopologicalSorter.done`, further calls will return all new nodes that + have all their predecessors already processed until no more progress can be + made. + + Raises :exc:`ValueError` if called without calling + :meth:`~TopologicalSorter.prepare` previously. + + .. method:: stable_order() + + Returns an iterable of nodes in a stable topological order. Using this method + does not require to call :meth:`TopologicalSorter.prepare` or + :meth:`TopologicalSorter.done`. This method is equivalent to:: + + def static_order(self): + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) - Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` previously. + If any cycle is detected, :exc:`CycleError` will be raised. .. versionadded:: 3.9 diff --git a/Lib/functools.py b/Lib/functools.py index dc37de74ec611a..522537314b8bbc 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -10,8 +10,8 @@ # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', - 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', - 'partialmethod', 'singledispatch', 'singledispatchmethod'] + 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'TopologicalSorter', + 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod'] from abc import get_cache_token from collections import namedtuple @@ -199,6 +199,7 @@ def total_ordering(cls): _NODE_OUT = -1 _NODE_DONE = -2 + class _NodeInfo: __slots__ = 'node', 'npredecessors', 'successors' @@ -215,25 +216,27 @@ def __init__(self, node): # long as they're all reflected in the successor's npredecessors attribute). self.successors = [] + class CycleError(ValueError): pass + class TopologicalSorter: """Provides functionality to topologically sort a graph of hashable nodes""" def __init__(self, graph=None): - self.node2info = {} - self.ready_nodes = None - self.npassedout = 0 - self.nfinished = 0 + self._node2info = {} + self._ready_nodes = None + self._npassedout = 0 + self._nfinished = 0 if graph is not None: for node, predecessors in graph.items(): self.add(node, *predecessors) def _get_nodeinfo(self, node): - if (result := self.node2info.get(node)) is None: - self.node2info[node] = result = _NodeInfo(node) + if (result := self._node2info.get(node)) is None: + self._node2info[node] = result = _NodeInfo(node) return result def add(self, node, *predecessors): @@ -251,7 +254,7 @@ def add(self, node, *predecessors): Raises ValueError if called after "prepare". """ - if self.ready_nodes is not None: + if self._ready_nodes is not None: raise ValueError("Nodes cannot be added after a call to prepare()") # Create the node -> predecessor edges @@ -271,11 +274,11 @@ def prepare(self): progress. After a call to this function, the graph cannot be modified and therefore no more nodes can be added using "add". """ - if self.ready_nodes is not None: + if self._ready_nodes is not None: raise ValueError("cannot prepare() more than once") - self.ready_nodes = [i.node for i in self.node2info.values() - if i.npredecessors == 0] + self._ready_nodes = [i.node for i in self._node2info.values() + if i.npredecessors == 0] # ready_nodes is set before we look for cycles on purpose: # if the user wants to catch the CycleError, that's fine, # they can continue using the instance to grab as many @@ -294,19 +297,19 @@ def get_ready(self): Raises ValueError if called without calling "prepare" previously. """ - if self.ready_nodes is None: + if self._ready_nodes is None: raise ValueError("prepare() must be called first") # Get the nodes that are ready and mark them - result = tuple(self.ready_nodes) - n2i = self.node2info + result = tuple(self._ready_nodes) + n2i = self._node2info for node in result: n2i[node].npredecessors = _NODE_OUT # Clean the list of nodes that are ready and update # the counter of nodes that we have returned. - self.ready_nodes.clear() - self.npassedout += len(result) + self._ready_nodes.clear() + self._npassedout += len(result) return result @@ -320,9 +323,9 @@ def is_active(self): Raises ValueError if called without calling "prepare" previously. """ - if self.ready_nodes is None: + if self._ready_nodes is None: raise ValueError("prepare() must be called first") - return self.nfinished < self.npassedout or bool(self.ready_nodes) + return self._nfinished < self._npassedout or bool(self._ready_nodes) def __bool__(self): return self.is_active() @@ -339,10 +342,10 @@ def done(self, *nodes): node has not yet been returned by "get_ready". """ - if self.ready_nodes is None: + if self._ready_nodes is None: raise ValueError("prepare() must be called first") - n2i = self.node2info + n2i = self._node2info for node in nodes: @@ -369,11 +372,11 @@ def done(self, *nodes): successor_info = n2i[successor] successor_info.npredecessors -= 1 if successor_info.npredecessors == 0: - self.ready_nodes.append(successor) - self.nfinished += 1 + self._ready_nodes.append(successor) + self._nfinished += 1 def _find_cycle(self): - n2i = self.node2info + n2i = self._node2info stack = [] itstack = [] seen = set() @@ -409,6 +412,19 @@ def _find_cycle(self): break return None + def static_order(self): + """Returns an iterable of nodes in a stable topological order. + + Using this method does not require to call "prepare" or "done". If any + cycle is detected, :exc:`CycleError` will be raised. + """ + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) + + ################################################################################ ### cmp_to_key() function converter ################################################################################ diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 81e4d9fdaa8516..6b5ca55723c406 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3,7 +3,7 @@ import collections import collections.abc import copy -from itertools import permutations +from itertools import permutations, chain import pickle from random import choice import sys @@ -1162,7 +1162,7 @@ class TestTopologicalSort(unittest.TestCase): def _test_graph(self, graph, expected): - def static_order(ts): + def static_order_with_groups(ts): ts.prepare() while ts.is_active(): nodes = ts.get_ready() @@ -1171,7 +1171,10 @@ def static_order(ts): yield nodes ts = functools.TopologicalSorter(graph) - self.assertEqual(list(static_order(ts)), expected) + self.assertEqual(list(static_order_with_groups(ts)), list(expected)) + + ts = functools.TopologicalSorter(graph) + self.assertEqual(list(ts.static_order()), list(chain(*expected))) def _assert_cycle(self, graph, cycle): ts = functools.TopologicalSorter() @@ -1198,6 +1201,9 @@ def test_simple_cases(self): self._test_graph({1: {}}, [(1,)]) + self._test_graph({x: {x+1} for x in range(10)}, + [(x,) for x in range(10, -1, -1)]) + def test_no_dependencies(self): self._test_graph( {1: {2}, From d0ba05bd658f47730c37eb8d5e9e68c1bb85a9b4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 17 Jan 2020 22:11:36 +0000 Subject: [PATCH 10/29] Misc improvements to the docs --- Doc/library/functools.rst | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 5470aef5dfe786..02b527b5a62c35 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -8,10 +8,16 @@ .. moduleauthor:: Raymond Hettinger .. moduleauthor:: Nick Coghlan .. moduleauthor:: Łukasz Langa +.. moduleauthor:: Pablo Galindo .. sectionauthor:: Peter Harris **Source code:** :source:`Lib/functools.py` +.. testsetup:: default + + import functools + from functools import * + -------------- The :mod:`functools` module is for higher-order functions: functions that act on @@ -546,7 +552,10 @@ The :mod:`functools` module defines the following functions: no paralellism is involved, the convencience method :meth:`TopologicalSorter.stable_oder` can be used directly. For example, using this method is possible to implement a simple version of the C3 linearization algorithm used by Python to calculate the Method - Resolution Order (MRO) of a derived class:: + Resolution Order (MRO) of a derived class: + + .. doctest:: + :hide: >>> class A: pass >>> class B(A): pass @@ -554,12 +563,13 @@ The :mod:`functools` module defines the following functions: >>> class D(B, C): pass >>> D.__mro__ - (__main__.D, __main__.B, __main__.C, __main__.A, object) + (, , , , ) + >>> graph = {D: {B, C}, C: {A}, B: {A}, A:{object}} >>> ts = TopologicalSorter(graph) - >>> topological_order = tuple(ts.stable_order()) + >>> topological_order = tuple(ts.static_order()) >>> tuple(reversed(topological_order)) - (__main__.D, '__main__.B, __main__.C, __main__.A, object) + (, , , , ) The class is designed to easily support parallel processing of the nodes as they become ready. For instance:: @@ -651,7 +661,7 @@ The :mod:`functools` module defines the following functions: Raises :exc:`ValueError` if called without calling :meth:`~TopologicalSorter.prepare` previously. - .. method:: stable_order() + .. method:: static_order() Returns an iterable of nodes in a stable topological order. Using this method does not require to call :meth:`TopologicalSorter.prepare` or From 2309642b5f5c3f9f49ca9d1468d1e455c7959d61 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 01:29:33 +0000 Subject: [PATCH 11/29] Apply suggestions from code review Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 02b527b5a62c35..dacc882903780d 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -549,8 +549,8 @@ The :mod:`functools` module defines the following functions: it finishes processing. In case that just an inmediate sorting of the nodes in the graph is required and - no paralellism is involved, the convencience method :meth:`TopologicalSorter.stable_oder` - can be used directly. For example, using this method is possible to implement a simple + no parallelism is involved, the convencience method :meth:`TopologicalSorter.stable_oder` + can be used directly. For example, this method can be used to implement a simple version of the C3 linearization algorithm used by Python to calculate the Method Resolution Order (MRO) of a derived class: From 3980ef86f9ebf84c2ce64685400f69112c433761 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 01:29:55 +0000 Subject: [PATCH 12/29] Apply suggestions from code review Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index dacc882903780d..e9db170fbc04b2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -542,7 +542,7 @@ The :mod:`functools` module defines the following functions: * Create an instance of the :class:`TopologicalSorter` with an optional initial graph. * Add additional nodes to the graph. - * Call "finalize()" on the graph. + * Call "prepare()" on the graph. * While :meth:`~TopologicalSorter.is_active` is ``True``: * Iterate over the nodes returned by :meth:`~TopologicalSorter.get_ready` and process them. Call :meth:`~TopologicalSorter.done` on each node as From 050c2bbc3f9745f620a244b725ee220132cab3cc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 02:19:41 +0000 Subject: [PATCH 13/29] Update Doc/library/functools.rst Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index e9db170fbc04b2..d33c3e8b047ecc 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -549,7 +549,7 @@ The :mod:`functools` module defines the following functions: it finishes processing. In case that just an inmediate sorting of the nodes in the graph is required and - no parallelism is involved, the convencience method :meth:`TopologicalSorter.stable_oder` + no parallelism is involved, the convenience method :meth:`TopologicalSorter.static_order` can be used directly. For example, this method can be used to implement a simple version of the C3 linearization algorithm used by Python to calculate the Method Resolution Order (MRO) of a derived class: From 11496852524bf3a0081e15b07763b1bada839bb8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 02:46:18 +0000 Subject: [PATCH 14/29] Update Doc/library/functools.rst Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index d33c3e8b047ecc..c40c9572bddef8 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -542,7 +542,7 @@ The :mod:`functools` module defines the following functions: * Create an instance of the :class:`TopologicalSorter` with an optional initial graph. * Add additional nodes to the graph. - * Call "prepare()" on the graph. + * Call :meth:`~TopologicalSorter.prepare` on the graph. * While :meth:`~TopologicalSorter.is_active` is ``True``: * Iterate over the nodes returned by :meth:`~TopologicalSorter.get_ready` and process them. Call :meth:`~TopologicalSorter.done` on each node as From cf8a729e460d7cc02ecde397debc55de3b12f80d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 02:49:41 +0000 Subject: [PATCH 15/29] Remove stable from the docs --- Doc/library/functools.rst | 2 +- Lib/functools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index c40c9572bddef8..c04c39e8230d9a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -663,7 +663,7 @@ The :mod:`functools` module defines the following functions: .. method:: static_order() - Returns an iterable of nodes in a stable topological order. Using this method + Returns an iterable of nodes in a topological order. Using this method does not require to call :meth:`TopologicalSorter.prepare` or :meth:`TopologicalSorter.done`. This method is equivalent to:: diff --git a/Lib/functools.py b/Lib/functools.py index 522537314b8bbc..1de23ba299216e 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -413,7 +413,7 @@ def _find_cycle(self): return None def static_order(self): - """Returns an iterable of nodes in a stable topological order. + """Returns an iterable of nodes in a topological order. Using this method does not require to call "prepare" or "done". If any cycle is detected, :exc:`CycleError` will be raised. From 26c52a6931dc5666f3d92224bcc1b0153e87f6dd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 02:55:03 +0000 Subject: [PATCH 16/29] Update Doc/library/functools.rst Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index c04c39e8230d9a..84781ceed143c7 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -586,7 +586,13 @@ The :mod:`functools` module defines the following functions: task_queue.put(node) # When the work for a node is done, workers put the node in - # 'finalized_tasks_queue' so we can get more nodes to work on + # 'finalized_tasks_queue' so we can get more nodes to work on. + # The definition of 'is_active()` guarantees that, at this point, at + # least one node has been placed on 'task_queue' that hasn't yet + # been passed to `done()`, so this blocking `get()` must (eventually) + # succeed. After calling `done()`, we loop back to call `get_ready()` + # again, so put newly freed nodes on 'task_queue' as soon as + # logically possible. node = finalized_tasks_queue.get() topological_sorter.done(node) From 4648f50d264c03b87b56608b1e28a0c802a6c7eb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 02:55:41 +0000 Subject: [PATCH 17/29] Update Doc/library/functools.rst Co-Authored-By: Tim Peters --- Doc/library/functools.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 84781ceed143c7..ccc2fc15e4d8d3 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -594,7 +594,6 @@ The :mod:`functools` module defines the following functions: # again, so put newly freed nodes on 'task_queue' as soon as # logically possible. node = finalized_tasks_queue.get() - topological_sorter.done(node) .. method:: add(node, *predecessors) From 3e2f2d9439f478791402a1ea7812d3f0ab098432 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 03:07:58 +0000 Subject: [PATCH 18/29] Fix quotes --- Doc/library/functools.rst | 6 +++--- Doc/myfile.bz2 | Bin 0 -> 331 bytes 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 Doc/myfile.bz2 diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index ccc2fc15e4d8d3..caf716a48e0677 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -587,10 +587,10 @@ The :mod:`functools` module defines the following functions: # When the work for a node is done, workers put the node in # 'finalized_tasks_queue' so we can get more nodes to work on. - # The definition of 'is_active()` guarantees that, at this point, at + # The definition of 'is_active()' guarantees that, at this point, at # least one node has been placed on 'task_queue' that hasn't yet - # been passed to `done()`, so this blocking `get()` must (eventually) - # succeed. After calling `done()`, we loop back to call `get_ready()` + # been passed to 'done()', so this blocking 'get()' must (eventually) + # succeed. After calling 'done()', we loop back to call 'get_ready()' # again, so put newly freed nodes on 'task_queue' as soon as # logically possible. node = finalized_tasks_queue.get() diff --git a/Doc/myfile.bz2 b/Doc/myfile.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..7ada20f60926b48c5e828d1e7bf71413ab4e70e9 GIT binary patch literal 331 zcmV-R0kr-?T4*^jL0KkKS+!R|D*ym2SAYNzKm{!$Kmb4Y{{S!nQ)N(SCV(fDn4n-m zsj1*o)jw1{KzfI%pc>6yC8lH}rXzFA}n%<_UC%QtSrIv)E4R zDh^1A%zp>6B3vmTgeYY0oJp0(`BpUNh|T3{*-_`)^ao&) zc!V(so1042t!WwNmV`neq?w3@G%ivk5jbims09KR+7&i4b+DaUp9+qJE21@v>K~jq d&ZNWD>qxYMPv6On5RHF}xgwk>NLs6)m4J*en85%5 literal 0 HcmV?d00001 From d82f697e2f1e2a8805dc43be74781ae9fbe95955 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 03:43:30 +0000 Subject: [PATCH 19/29] Add more test cases --- Lib/test/test_functools.py | 82 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6b5ca55723c406..1b2fcc21892c1b 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -13,9 +13,12 @@ import typing import unittest import unittest.mock +import os from weakref import proxy import contextlib +from test.support.script_helper import assert_python_ok + import functools py_functools = support.import_fresh_module('functools', blocked=['_functools']) @@ -1204,6 +1207,46 @@ def test_simple_cases(self): self._test_graph({x: {x+1} for x in range(10)}, [(x,) for x in range(10, -1, -1)]) + self._test_graph({2: {3}, 3: {4}, 4: {5}, 5: {1}, + 11: {12}, 12: {13}, 13: {14}, 14: {15}}, + [(1, 15), (5, 14), (4, 13), (3, 12), (2, 11)]) + + self._test_graph({ + 0: [1, 2], + 1: [3], + 2: [5, 6], + 3: [4], + 4: [9], + 5: [3], + 6: [7], + 7: [8], + 8: [4], + 9: [] + }, + [(9,), (4,), (3, 8), (1, 5, 7), (6,), (2,), (0,)] + ) + + self._test_graph({ + 0: [1, 2], + 1: [], + 2: [3], + 3: [] + }, + [(1, 3), (2,), (0,)] + ) + + self._test_graph({ + 0: [1, 2], + 1: [], + 2: [3], + 3: [], + 4: [5], + 5: [6], + 6: [] + }, + [(1, 3, 6), (2, 5), (0, 4)] + ) + def test_no_dependencies(self): self._test_graph( {1: {2}, @@ -1308,6 +1351,45 @@ def test_not_hashable_nodes(self): self.assertRaises(TypeError, ts.add, 1, dict()) self.assertRaises(TypeError, ts.add, dict(), dict()) + def test_order_of_insertion_does_not_matter(self): + ts = functools.TopologicalSorter() + ts.add(3, 2, 1) + ts.add(2, 1) + ts.add(1, 0) + + ts2 = functools.TopologicalSorter() + ts2.add(2, 1) + ts2.add(1, 0) + ts2.add(3, 2, 1) + + self.assertEqual([*ts.static_order()], [*ts2.static_order()]) + + def test_static_order_does_not_change_with_the_hash_seed(self): + def check_order_with_hash_seed(seed): + code = """if 1: + import functools + ts = functools.TopologicalSorter() + ts.add(1, 2, 3, 4, 5) + ts.add(2, 3, 4, 5, 6) + ts.add(4, 11, 45, 3) + ts.add(0, 11, 2, 3, 4) + print(list(ts.static_order())) + """ + env = os.environ.copy() + # signal to assert_python not to do a copy + # of os.environ on its own + env['__cleanenv'] = True + env['PYTHONHASHSEED'] = str(seed) + out = assert_python_ok('-c', code, **env) + return out + + run1 = check_order_with_hash_seed(1234) + run2 = check_order_with_hash_seed(31415) + + self.assertNotEqual(run1, "") + self.assertNotEqual(run2, "") + self.assertEqual(run1, run2) + class TestLRU: From c0d2ff69207f0dd5ed371b1e238d4659e53f26f4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 04:05:59 +0000 Subject: [PATCH 20/29] Add even more test cases --- Lib/test/test_functools.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 1b2fcc21892c1b..ffa8d292a68c68 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1262,6 +1262,34 @@ def test_no_dependencies(self): [(1, 3, 5)] ) + def test_the_node_multiple_times(self): + # Test same node multiple times in dependencies + self._test_graph({1: {2}, 3: {4}, 0: [2, 4, 4, 4, 4, 4]}, + [(2, 4), (1, 3, 0)]) + + # Test adding the same dependency multiple times + ts = functools.TopologicalSorter() + ts.add(1, 2) + ts.add(1, 2) + ts.add(1, 2) + self.assertEqual([*ts.static_order()], [2, 1]) + + def test_graph_with_iterables(self): + dependson = (2*x + 1 for x in range(5)) + ts = functools.TopologicalSorter({0: dependson}) + self.assertEqual(list(ts.static_order()), [1, 3, 5, 7, 9, 0]) + + def test_add_dependencies_for_same_node_incrementally(self): + # Test same node multiple times + ts = functools.TopologicalSorter() + ts.add(1, 2) + ts.add(1, 3) + ts.add(1, 4) + ts.add(1, 5) + + ts2 = functools.TopologicalSorter({1: {2, 3, 4, 5}}) + self.assertEqual([*ts.static_order()], [*ts2.static_order()]) + def test_empty(self): self._test_graph({}, []) From ed8c4eeffacf3b8f33228f114313e5dd755f287c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 05:13:15 +0000 Subject: [PATCH 21/29] Use strings for the hash test and compare groups for insertion order --- Lib/test/test_functools.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ffa8d292a68c68..553b3ef95cf3a9 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1379,28 +1379,43 @@ def test_not_hashable_nodes(self): self.assertRaises(TypeError, ts.add, 1, dict()) self.assertRaises(TypeError, ts.add, dict(), dict()) - def test_order_of_insertion_does_not_matter(self): + def test_order_of_insertion_does_not_matter_between_groups(self): + def get_groups(ts): + ts.prepare() + while ts.is_active(): + nodes = ts.get_ready() + for node in nodes: + ts.done(node) + yield nodes + ts = functools.TopologicalSorter() ts.add(3, 2, 1) - ts.add(2, 1) ts.add(1, 0) + ts.add(4, 5) + ts.add(6, 7) + ts.add(4, 7) ts2 = functools.TopologicalSorter() - ts2.add(2, 1) ts2.add(1, 0) ts2.add(3, 2, 1) + ts2.add(4, 7) + ts2.add(6, 7) + ts2.add(4, 5) - self.assertEqual([*ts.static_order()], [*ts2.static_order()]) + self.assertEqual( + list(map(set, get_groups(ts))), + list(map(set, get_groups(ts2))) + ) def test_static_order_does_not_change_with_the_hash_seed(self): def check_order_with_hash_seed(seed): code = """if 1: import functools ts = functools.TopologicalSorter() - ts.add(1, 2, 3, 4, 5) - ts.add(2, 3, 4, 5, 6) - ts.add(4, 11, 45, 3) - ts.add(0, 11, 2, 3, 4) + ts.add('blech', 'bluch', 'hola') + ts.add('abcd', 'blech', 'bluch', 'a', 'b') + ts.add('a', 'a string', 'something', 'b') + ts.add('bluch', 'hola', 'abcde', 'a', 'b') print(list(ts.static_order())) """ env = os.environ.copy() From 6229ad5df5966708071013fd78b5976648af42c6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 05:26:20 +0000 Subject: [PATCH 22/29] Add example on insertion order affecting static_order() --- Doc/library/functools.rst | 23 +++++++++++++++++++++++ Lib/functools.py | 3 +++ 2 files changed, 26 insertions(+) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index caf716a48e0677..0d3712264ac634 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -679,6 +679,29 @@ The :mod:`functools` module defines the following functions: yield from node_group self.done(*node_group) + The particular order that is returned may depend on the particular order in + which the items were inserted in the graph. For example: + + .. doctest:: + :hide: + + >>> ts = TopologicalSorter() + >>> ts.add(3, 2, 1) + >>> ts.add(1, 0) + >>> print([*ts.static_order()]) + [2, 0, 1, 3] + + >>> ts2 = TopologicalSorter() + >>> ts2.add(1, 0) + >>> ts2.add(3, 2, 1) + >>> print([*ts2.static_order()]) + [0, 2, 1, 3] + + This is due to the fact that "0" and "2" are in the same level in the graph (they + would have been returned in the same call to :meth:`~TopologicalSorter.get_ready`) + and the order between them is determined by the order of insertion. + + If any cycle is detected, :exc:`CycleError` will be raised. .. versionadded:: 3.9 diff --git a/Lib/functools.py b/Lib/functools.py index 1de23ba299216e..262fe9700af69f 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -415,6 +415,9 @@ def _find_cycle(self): def static_order(self): """Returns an iterable of nodes in a topological order. + The particular order that is returned may depend on the particular + order in which the items were inserted in the graph. + Using this method does not require to call "prepare" or "done". If any cycle is detected, :exc:`CycleError` will be raised. """ From 618a7913ea4da2b695c3df993be66329e7cf1e51 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 05:33:08 +0000 Subject: [PATCH 23/29] Simplify test --- Doc/library/functools.rst | 2 +- Lib/functools.py | 2 +- Lib/test/test_functools.py | 10 +++------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 0d3712264ac634..1140e1e40615d3 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -679,7 +679,7 @@ The :mod:`functools` module defines the following functions: yield from node_group self.done(*node_group) - The particular order that is returned may depend on the particular order in + The particular order that is returned may depend on the specific order in which the items were inserted in the graph. For example: .. doctest:: diff --git a/Lib/functools.py b/Lib/functools.py index 262fe9700af69f..9aee00cf533699 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -415,7 +415,7 @@ def _find_cycle(self): def static_order(self): """Returns an iterable of nodes in a topological order. - The particular order that is returned may depend on the particular + The particular order that is returned may depend on the specific order in which the items were inserted in the graph. Using this method does not require to call "prepare" or "done". If any diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 553b3ef95cf3a9..6b3d53bb5d5ac8 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1384,9 +1384,8 @@ def get_groups(ts): ts.prepare() while ts.is_active(): nodes = ts.get_ready() - for node in nodes: - ts.done(node) - yield nodes + ts.done(*nodes) + yield set(nodes) ts = functools.TopologicalSorter() ts.add(3, 2, 1) @@ -1402,10 +1401,7 @@ def get_groups(ts): ts2.add(6, 7) ts2.add(4, 5) - self.assertEqual( - list(map(set, get_groups(ts))), - list(map(set, get_groups(ts2))) - ) + self.assertEqual(list(get_groups(ts)), list(get_groups(ts2))) def test_static_order_does_not_change_with_the_hash_seed(self): def check_order_with_hash_seed(seed): From 1d926ab5fbf198912018604544a495cbc1901938 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 19:40:10 +0000 Subject: [PATCH 24/29] Apply suggestions from code review Co-Authored-By: Tim Peters --- Lib/functools.py | 3 ++- Lib/test/test_functools.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 9aee00cf533699..f0ff55d31f2010 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -10,7 +10,8 @@ # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', - 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'TopologicalSorter', + 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', + 'TopologicalSorter', 'CycleError', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod'] from abc import get_cache_token diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6b3d53bb5d5ac8..9503f4086b1cb9 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1308,7 +1308,7 @@ def test_cycle(self): # Cycle in the middle of the graph self._assert_cycle({1: {2}, 2: {3}, 3: {2, 4}, 4: {5}}, [3, 2]) - def test_calls_before_preapare(self): + def test_calls_before_prepare(self): ts = functools.TopologicalSorter() with self.assertRaisesRegex(ValueError, r"prepare\(\) must be called first"): From 064209a0ce7534cfef5c1c31f6ebf62c9d6c6a46 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 18 Jan 2020 22:54:29 +0000 Subject: [PATCH 25/29] Improvements to the documentation formatting --- Doc/library/functools.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1140e1e40615d3..bcf92e081e59d9 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -539,14 +539,12 @@ The :mod:`functools` module defines the following functions: In the general case, the steps required to perform the sorting of a given graph are as follows: - * Create an instance of the :class:`TopologicalSorter` with an optional - initial graph. + * Create an instance of the :class:`TopologicalSorter` with an optional initial graph. * Add additional nodes to the graph. * Call :meth:`~TopologicalSorter.prepare` on the graph. - * While :meth:`~TopologicalSorter.is_active` is ``True``: - * Iterate over the nodes returned by :meth:`~TopologicalSorter.get_ready` - and process them. Call :meth:`~TopologicalSorter.done` on each node as - it finishes processing. + * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over the + nodes returned by :meth:`~TopologicalSorter.get_ready` and process them. + Call :meth:`~TopologicalSorter.done` on each node as it finishes processing. In case that just an inmediate sorting of the nodes in the graph is required and no parallelism is involved, the convenience method :meth:`TopologicalSorter.static_order` @@ -555,7 +553,6 @@ The :mod:`functools` module defines the following functions: Resolution Order (MRO) of a derived class: .. doctest:: - :hide: >>> class A: pass >>> class B(A): pass @@ -683,7 +680,6 @@ The :mod:`functools` module defines the following functions: which the items were inserted in the graph. For example: .. doctest:: - :hide: >>> ts = TopologicalSorter() >>> ts.add(3, 2, 1) From cc7aacd0fb4efdb5ae448c03516174a6050f70ac Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 20 Jan 2020 09:56:07 +0000 Subject: [PATCH 26/29] Apply suggestions from code review Co-Authored-By: sweeneyde <36520290+sweeneyde@users.noreply.github.com> --- Doc/library/functools.rst | 13 +++++++------ Lib/functools.py | 5 +++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index bcf92e081e59d9..5f3bd1cf85d318 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -531,7 +531,7 @@ The :mod:`functools` module defines the following functions: the graph has no directed cycles, that is, if it is a directed acyclic graph. If the optional *graph* argument is provided it must be a dictionary representing - a direct acyclic graph where the keys are nodes and the values are iterables of + a directed acyclic graph where the keys are nodes and the values are iterables of all predecessors of that node in the graph (the nodes that have edges that point to the value in the key). Additional nodes can be added to the graph using the :meth:`~TopologicalSorter.add` method. @@ -546,7 +546,7 @@ The :mod:`functools` module defines the following functions: nodes returned by :meth:`~TopologicalSorter.get_ready` and process them. Call :meth:`~TopologicalSorter.done` on each node as it finishes processing. - In case that just an inmediate sorting of the nodes in the graph is required and + In case just an immediate sorting of the nodes in the graph is required and no parallelism is involved, the convenience method :meth:`TopologicalSorter.static_order` can be used directly. For example, this method can be used to implement a simple version of the C3 linearization algorithm used by Python to calculate the Method @@ -602,7 +602,7 @@ The :mod:`functools` module defines the following functions: will be the union of all dependencies passed in. It is possible to add a node with no dependencies (*predecessors* is not - provided) as well as provide a dependency twice. If a node that has not been + provided) or to provide a dependency twice. If a node that has not been provided before is included among *predecessors* it will be automatically added to the graph with no predecessors of its own. @@ -614,7 +614,7 @@ The :mod:`functools` module defines the following functions: detected, :exc:`CycleError` will be raised, but :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many nodes as possible until cycles block more progress. After a call to this function, - the graph cannot be modified and therefore no more nodes can be added using + the graph cannot be modified, and therefore no more nodes can be added using :meth:`~TopologicalSorter.add`. .. method:: is_active() @@ -655,9 +655,10 @@ The :mod:`functools` module defines the following functions: .. method:: get_ready() Returns a ``tuple`` with all the nodes that are ready. Initially it returns all - nodes with no predecessors and once those are marked as processed by calling + nodes with no predecessors, and once those are marked as processed by calling :meth:`TopologicalSorter.done`, further calls will return all new nodes that - have all their predecessors already processed until no more progress can be + have all their predecessors already processed. Once no more progress can be + made, empty tuples are returned. made. Raises :exc:`ValueError` if called without calling diff --git a/Lib/functools.py b/Lib/functools.py index f0ff55d31f2010..b7f5737be844d5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -291,9 +291,10 @@ def prepare(self): def get_ready(self): """Return a tuple of all the nodes that are ready. - Initially it returns all nodes with no predecessors and once those are marked + Initially it returns all nodes with no predecessors; once those are marked as processed by calling "done", further calls will return all new nodes that - have all their predecessors already processed until no more progress can be + have all their predecessors already processed. Once no more progress can be made, + empty tuples are returned. made. Raises ValueError if called without calling "prepare" previously. From c82d41b34f253c51d28f8c5adc5473a522d3662c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 20 Jan 2020 18:13:22 +0000 Subject: [PATCH 27/29] Correct typo --- Lib/functools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index b7f5737be844d5..5cf05a4f2764e2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -295,7 +295,6 @@ def get_ready(self): as processed by calling "done", further calls will return all new nodes that have all their predecessors already processed. Once no more progress can be made, empty tuples are returned. - made. Raises ValueError if called without calling "prepare" previously. """ From 81c1833cdbdfb389f4b69e944724d0d9c6a96730 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 23 Jan 2020 14:54:02 +0000 Subject: [PATCH 28/29] Document CycleError --- Doc/library/functools.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 5f3bd1cf85d318..8c408923b70a72 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -813,3 +813,19 @@ differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods during instance attribute look-up. + + +Exceptions +---------- +The :mod:`functools` module defines the following exception classes: + +.. exception:: CycleError + + Subclass of :exc:`ValueError` raised by :meth:`TopologicalSorter.prepare` if cycles exist + in the working graph. If multiple cycles exist, only one undefined choice among them will + be reported and included in the exception. + + The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` + attribute of the exception instance and consists in a list of nodes, such that each node is, + in the graph, an immediate predecessor of the next node in the list. In the reported list, + the first and the last node will be the same, to make it clear that it is cyclic. From 89d07cec28ac7b4cb54493c8d2154730fd0696fb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 23 Jan 2020 14:58:10 +0000 Subject: [PATCH 29/29] fixup! Document CycleError --- Lib/functools.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/functools.py b/Lib/functools.py index 5cf05a4f2764e2..050bec86051179 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -219,6 +219,15 @@ def __init__(self, node): class CycleError(ValueError): + """Subclass of ValueError raised by TopologicalSorterif cycles exist in the graph + + If multiple cycles exist, only one undefined choice among them will be reported + and included in the exception. The detected cycle can be accessed via the second + element in the *args* attribute of the exception instance and consists in a list + of nodes, such that each node is, in the graph, an immediate predecessor of the + next node in the list. In the reported list, the first and the last node will be + the same, to make it clear that it is cyclic. + """ pass