diff --git a/connect/src/main/protobuf/graphframes.proto b/connect/src/main/protobuf/graphframes.proto index 07930fa9c..489f2585b 100644 --- a/connect/src/main/protobuf/graphframes.proto +++ b/connect/src/main/protobuf/graphframes.proto @@ -40,6 +40,7 @@ message GraphFramesAPI { RandomWalkEmbeddings rw_embeddings = 23; AggregateNeighbors aggregate_neighbors = 24; NeighborhoodAwareCDLP neighborhood_aware_cdlp = 25; + AllPaths all_paths = 26; } } @@ -88,6 +89,17 @@ message BFS { int32 max_path_length = 4; } +message AllPaths { + ColumnOrExpression from_expr = 1; + ColumnOrExpression to_expr = 2; + ColumnOrExpression edge_filter = 3; + int32 max_path_length = 4; + bool is_directed = 5; + int32 checkpoint_interval = 6; + bool use_local_checkpoints = 7; + optional StorageLevel storage_level = 8; +} + message ConnectedComponents { string algorithm = 1; int32 checkpoint_interval = 2; diff --git a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala index fbd875470..887022048 100644 --- a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala +++ b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala @@ -202,6 +202,17 @@ object GraphFramesConnectUtils { .maxPathLength(bfsProto.getMaxPathLength) .run() } + case proto.GraphFramesAPI.MethodCase.ALL_PATHS => { + val allPathsProto = apiMessage.getAllPaths + graphFrame.allPaths + .toExpr(parseColumnOrExpression(allPathsProto.getToExpr, planner)) + .fromExpr(parseColumnOrExpression(allPathsProto.getFromExpr, planner)) + .edgeFilter(parseColumnOrExpression(allPathsProto.getEdgeFilter, planner)) + .maxPathLength(allPathsProto.getMaxPathLength) + .setIsDirected(allPathsProto.getIsDirected) + .setUseLocalCheckpoints(allPathsProto.getUseLocalCheckpoints) + .run() + } case proto.GraphFramesAPI.MethodCase.CONNECTED_COMPONENTS => { val cc = apiMessage.getConnectedComponents val ccBuilder = graphFrame.connectedComponents diff --git a/core/src/main/scala/org/graphframes/GraphFrame.scala b/core/src/main/scala/org/graphframes/GraphFrame.scala index a84a12e68..16bed7cac 100644 --- a/core/src/main/scala/org/graphframes/GraphFrame.scala +++ b/core/src/main/scala/org/graphframes/GraphFrame.scala @@ -701,6 +701,15 @@ class GraphFrame private ( */ def bfs: BFS = new BFS(this) + /** + * Enumerate all paths between source and destination vertices. + * + * See [[org.graphframes.lib.AllPaths]] for details. + * + * @group stdlib + */ + def allPaths: AllPaths = new AllPaths(this) + /** * Aggregate information from neighboring vertices and edges through a controlled traversal. * diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala new file mode 100644 index 000000000..279b8d01a --- /dev/null +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.graphframes.lib + +import org.apache.spark.sql.Column +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.array +import org.apache.spark.sql.functions.array_contains +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.functions.concat +import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.graphframes.SparkShims +import org.graphframes.GraphFrame +import org.graphframes.WithCheckpointInterval +import org.graphframes.WithDirection +import org.graphframes.WithIntermediateStorageLevel +import org.graphframes.WithLocalCheckpoints + +/** + * Computes all simple paths between source and destination vertices. + * + * This algorithm enumerates paths up to `maxPathLength` hops. It supports directed and undirected + * traversal as well as optional edge filtering. It returns all simple paths between source and + * destination vertices. Here the term "simple" means no repeated vertices. For example, if there + * are paths A-B-C, A-D-C and the edge B-A, user asked to find all the paths between "A" and "C" + * only A-B-C and A-D-C will be returned, but not the A-B-A-D-C. The default value of the + * `maxPathLength` is `5`. Keep in mind that requesting `maxPathLength` of the scale of the graph + * diameter may tend this algorithm will try to return (almost) all simple paths in the graph that + * can create huge performance degradation or even OOM-like errors. Algorithm supports both + * directed and undirected graphs. + * + * Returned DataFrame schema: + * - `path`: array of vertex ids in traversal order + * - `len`: number of edges in the path (Long) + * + * Note: in the case of undirected graph an algorithm run on the internal graph made by union + * edges and reversed edges. It is assummed that graph does not have multi-edges. Results may be + * unstable and unpredictable for the graph with multi-edges. + */ +class AllPaths private[graphframes] (private val graph: GraphFrame) + extends Arguments + with Serializable + with WithDirection + with WithLocalCheckpoints + with WithCheckpointInterval + with WithIntermediateStorageLevel { + + private var maxPathLength: Int = 5 + private var fromExpression: Column = _ + private var toExpression: Column = _ + private var edgeFilterExpression: Option[Column] = None + + /** + * Sets the expression identifying the source (starting) vertices. + * + * @param value + * a Column expression evaluated against vertex attributes to select source vertices + * @return + * this instance for method chaining + */ + def fromExpr(value: Column): this.type = { + fromExpression = value + this + } + + /** + * Sets the expression identifying the source (starting) vertices. + * + * @param value + * a SQL expression string evaluated against vertex attributes to select source vertices + * @return + * this instance for method chaining + */ + def fromExpr(value: String): this.type = fromExpr(expr(value)) + + /** + * Sets the expression identifying the destination (target) vertices. + * + * @param value + * a Column expression evaluated against vertex attributes to select destination vertices + * @return + * this instance for method chaining + */ + def toExpr(value: Column): this.type = { + toExpression = value + this + } + + /** + * Sets the expression identifying the destination (target) vertices. + * + * @param value + * a SQL expression string evaluated against vertex attributes to select destination vertices + * @return + * this instance for method chaining + */ + def toExpr(value: String): this.type = toExpr(expr(value)) + + /** + * Sets the maximum path length (number of edges) for the enumerated paths. + * + * Setting a large value (e.g. on the scale of the graph diameter) may cause the algorithm to + * attempt to collect a very large number of paths, leading to severe performance degradation or + * out-of-memory errors. Use with caution on large or densely connected graphs. + * + * @param value + * the maximum number of edges in a path; must be greater than 0. Default is 5. + * @return + * this instance for method chaining + */ + def maxPathLength(value: Int): this.type = { + require(value > 0, s"AllPaths maxPathLength must be > 0, but was set to $value") + maxPathLength = value + this + } + + /** + * Sets an optional filter expression applied to edges during traversal. Only edges satisfying + * this condition will be considered. + * + * @param value + * a Column expression evaluated against edge attributes + * @return + * this instance for method chaining + */ + def edgeFilter(value: Column): this.type = { + edgeFilterExpression = Some(value) + this + } + + /** + * Sets an optional filter expression applied to edges during traversal. Only edges satisfying + * this condition will be considered. + * + * @param value + * a SQL expression string evaluated against edge attributes + * @return + * this instance for method chaining + */ + def edgeFilter(value: String): this.type = edgeFilter(expr(value)) + + /** + * Executes the AllPaths algorithm and returns all simple paths between the specified source and + * destination vertices. + * + * @return + * a DataFrame with the following columns: + * - `path`: an array of vertex ids in traversal order + * - `len`: the number of edges in the path (Long) + */ + def run(): DataFrame = { + require(fromExpression != null, "fromExpr is required.") + require(toExpression != null, "toExpr is required.") + require( + graph.vertices.columns.toSet.intersect(Set("hop", "path", "len")).isEmpty, + "columns `hop`, `path` and `len` are reserved by algorithm") + + val traversalGraph = if (isDirected) { + graph + } else { + val edgeColumns = graph.edges.columns.toSeq + val reversed = graph.edges.select( + (Seq( + col(GraphFrame.DST).alias(GraphFrame.SRC), + col(GraphFrame.SRC).alias(GraphFrame.DST)) ++ + edgeColumns.filterNot(c => c == GraphFrame.SRC || c == GraphFrame.DST).map(col)): _*) + GraphFrame(graph.vertices, graph.edges.unionByName(reversed)) + } + + val agg = traversalGraph.aggregateNeighbors + .setStartingVertices(fromExpression) + .setMaxHops(maxPathLength) + .setTargetCondition(SparkShims.applyExprToCol(graph.spark, toExpression, "dst_attributes")) + .setStoppingCondition( + array_contains(col("path"), AggregateNeighbors.dstAttr(GraphFrame.ID))) + .addAccumulator( + "path", + array(col(GraphFrame.ID)), + concat(col("path"), array(AggregateNeighbors.dstAttr(GraphFrame.ID)))) + .setUseLocalCheckpoints(useLocalCheckpoints) + .setCheckpointInterval(checkpointInterval) + .setIntermediateStorageLevel(intermediateStorageLevel) + + edgeFilterExpression.foreach { ef => + agg.setEdgeFilter(SparkShims.applyExprToCol(graph.spark, ef, "edge_attributes")) + } + + agg + .run() + .select(col("path"), col("hop").alias("len")) + .distinct() + } +} diff --git a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala new file mode 100644 index 000000000..4e3ef85a5 --- /dev/null +++ b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.graphframes.lib + +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.ArrayType +import org.apache.spark.sql.types.IntegerType +import org.apache.spark.sql.types.LongType +import org.graphframes.GraphFrame +import org.graphframes.GraphFrameTestSparkContext +import org.graphframes.SparkFunSuite + +class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { + + test("directed: enumerate all simple paths") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = + spark + .createDataFrame(Seq((1L, 2L, "x"), (2L, 4L, "x"), (1L, 3L, "x"), (3L, 4L, "x"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = + g.allPaths.fromExpr(col("id") === 1L).toExpr(col("id") === 4L).maxPathLength(3).run() + + val actual = result + .collect() + .map { row => + row.getAs[Seq[Long]]("path") -> row.getAs[Long]("len") + } + .toSet + + val expected = Set((Seq(1L, 2L, 4L), 2L), (Seq(1L, 3L, 4L), 2L)) + assert(actual === expected) + } + + test("undirected: traverse both edge directions") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = + spark + .createDataFrame(Seq((1L, 2L, "x"), (2L, 4L, "x"), (1L, 3L, "x"), (3L, 4L, "x"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths + .fromExpr(col("id") === 4L) + .toExpr(col("id") === 1L) + .setIsDirected(false) + .maxPathLength(3) + .run() + + val actual = result + .collect() + .map { row => + row.getAs[Seq[Long]]("path") -> row.getAs[Long]("len") + } + .toSet + + val expected = Set((Seq(4L, 2L, 1L), 2L), (Seq(4L, 3L, 1L), 2L)) + assert(actual === expected) + } + + test("edge filter excludes blocked edges") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = spark + .createDataFrame( + Seq((1L, 2L, "allowed"), (2L, 4L, "allowed"), (1L, 3L, "blocked"), (3L, 4L, "allowed"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths + .fromExpr("id = 1") + .toExpr("id = 4") + .edgeFilter(col("label") =!= "blocked") + .maxPathLength(3) + .run() + + val rows = result.collect() + assert(rows.length === 1) + assert(rows.head.getAs[Seq[Long]]("path") === Seq(1L, 2L, 4L)) + assert(rows.head.getAs[Long]("len") === 2L) + } + + test("cycle handling keeps paths simple") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"))).toDF("id", "name") + val edges = spark + .createDataFrame(Seq((1L, 2L), (2L, 1L), (2L, 3L), (1L, 3L))) + .toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths.fromExpr("id = 1").toExpr("id = 3").maxPathLength(4).run() + + val actualPaths = + result.collect().map(_.getAs[scala.collection.Seq[Long]]("path").toList).toSet + assert(actualPaths === Set(List(1L, 3L), List(1L, 2L, 3L))) + assert(!actualPaths.contains(List(1L, 2L, 1L, 3L))) + } + + test("schema and required arguments") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"))).toDF("id", "name") + val edges = spark.createDataFrame(Seq((1L, 2L))).toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths.fromExpr("id = 1").toExpr("id = 2").run() + + assert(result.columns.toSeq === Seq("path", "len")) + assert(result.schema("path").dataType.asInstanceOf[ArrayType].elementType === LongType) + assert(result.schema("len").dataType === IntegerType) + + intercept[IllegalArgumentException] { + g.allPaths.run() + } + intercept[IllegalArgumentException] { + g.allPaths.fromExpr("id = 1").run() + } + intercept[IllegalArgumentException] { + g.allPaths.toExpr("id = 2").run() + } + intercept[IllegalArgumentException] { + g.allPaths.fromExpr("id = 1").toExpr("id = 2").maxPathLength(0) + } + } + + test("no matching path returns empty dataframe") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"))).toDF("id", "name") + val edges = spark.createDataFrame(Seq((1L, 2L))).toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = + g.allPaths.fromExpr(col("id") === 1L).toExpr(col("id") === 3L).maxPathLength(4).run() + assert(result.collect().isEmpty) + } +} diff --git a/docs/src/04-user-guide/05-traversals.md b/docs/src/04-user-guide/05-traversals.md index 9c8bc6d62..479b2e212 100644 --- a/docs/src/04-user-guide/05-traversals.md +++ b/docs/src/04-user-guide/05-traversals.md @@ -67,6 +67,104 @@ The level of storage for intermediate results and the output `DataFrame` with co By default this is true and algorithm will look for only directed paths. By passing false, graph will be considered as undirected and algorithm will look for any shortest path. +## All paths + +Computes all simple paths between source and destination vertices up to a specified maximum path length. A "simple" path means no repeated vertices — for example, if there are paths A-B-C and A-D-C and an edge B-A, querying for all paths between A and C returns A-B-C and A-D-C but not A-B-A-D-C. + +The algorithm supports both directed and undirected traversal as well as optional edge filtering. + +--- + +**WARNING:** + +_Depending on the value of `max_path_length`, this algorithm can be **extremely slow** or even lead to **out-of-memory (OOM) errors** or **huge disk spills**. Setting `max_path_length` on the scale of the graph diameter causes the algorithm to attempt to enumerate (almost) all simple paths in the graph, which grows combinatorially. Use with caution, especially on large or densely connected graphs. Start with a small value and increase gradually._ + +--- + +### Python API + +For API details, refer to the @:pydoc(graphframes.GraphFrame.all_paths). + +```python +from graphframes.examples import Graphs + +g = Graphs(spark).friends() # Get example graph + +# Find all simple paths from vertex "a" to vertex "d" with at most 3 edges +paths = g.all_paths( + from_expr="id = 'a'", + to_expr="id = 'd'", + max_path_length=3, +) +paths.select("path", "len").show() + +# Search with edge filter and undirected traversal +paths = g.all_paths( + from_expr="id = 'a'", + to_expr="id = 'd'", + edge_filter="relationship != 'friend'", + max_path_length=3, + is_directed=False, +) +paths.select("path", "len").show() +``` + +### Scala API + +For API details, refer to the @:scaladoc(org.graphframes.lib.AllPaths). + +```scala +import org.graphframes.{examples, GraphFrame} + +val g: GraphFrame = examples.Graphs.friends // get example graph + +// Find all simple paths from vertex "a" to vertex "d" with at most 3 edges +val paths = g.allPaths + .fromExpr("id = 'a'") + .toExpr("id = 'd'") + .maxPathLength(3) + .run() +paths.select("path", "len").show() + +// Search with edge filter +val pathsFiltered = g.allPaths + .fromExpr("id = 'a'") + .toExpr("id = 'd'") + .edgeFilter("relationship != 'friend'") + .maxPathLength(3) + .run() +pathsFiltered.select("path", "len").show() +``` + +### Returned DataFrame Schema + +The returned `DataFrame` contains the following columns: + +- `path`: an array of vertex IDs in traversal order (e.g., `[a, b, d]`) +- `len`: the number of edges in the path (Long) + +### Arguments + +- `from_expr` (Python) / `fromExpr` (Scala) + +A Column or SQL expression identifying the source (starting) vertices. + +- `to_expr` (Python) / `toExpr` (Scala) + +A Column or SQL expression identifying the destination (target) vertices. + +- `max_path_length` (Python) / `maxPathLength` (Scala) + +Maximum number of edges in a path; must be greater than 0. Default is 5. Setting a large value (e.g., on the scale of the graph diameter) may cause severe performance degradation or out-of-memory errors. + +- `edge_filter` (Python) / `edgeFilter` (Scala) + +An optional Column or SQL expression applied to edges during traversal. Only edges satisfying this condition are considered. + +- `is_directed` (Python) / directed mode (Scala) + +Whether to use directed traversal. Default is `True`. If `False`, the graph is treated as undirected by internally unioning edges with reversed edges. Note: it is assumed that the graph does not have multi-edges. Results may be unstable for graphs with multi-edges. + ## Breadth-first search (BFS) Breadth-first search (BFS) finds the shortest path(s) from one vertex (or a set of vertices) to another vertex (or a set @@ -455,6 +553,7 @@ This algorithm is particularly useful for: - **Accumulative Computations**: Computing values that depend on the entire path (e.g., product of edge weights, sum of node values) Unlike single-hop algorithms like BFS or shortest paths, Aggregate Neighbors allows you to: + - Track multiple accumulators simultaneously - Define custom stopping criteria - Access vertex and edge attributes during traversal @@ -464,7 +563,7 @@ Unlike single-hop algorithms like BFS or shortest paths, Aggregate Neighbors all **NOTE** -*Be aware, that returned `DataFrame` is persistent and should be unpersisted manually after processing to avoid memory leaks!* +_Be aware, that returned `DataFrame` is persistent and should be unpersisted manually after processing to avoid memory leaks!_ --- diff --git a/python/graphframes/classic/graphframe.py b/python/graphframes/classic/graphframe.py index 2f7169cd7..fe0ffdbbc 100644 --- a/python/graphframes/classic/graphframe.py +++ b/python/graphframes/classic/graphframe.py @@ -140,6 +140,42 @@ def bfs( jdf = builder.run() return DataFrame(jdf, self._spark) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, + ) -> DataFrame: + builder = self._jvm_graph.allPaths() + if isinstance(from_expr, Column): + builder.fromExpr(from_expr._jc) + else: + builder.fromExpr(from_expr) + if isinstance(to_expr, Column): + builder.toExpr(to_expr._jc) + else: + builder.toExpr(to_expr) + builder.maxPathLength(max_path_length).setIsDirected(is_directed) + if edge_filter is not None: + if isinstance(edge_filter, Column): + builder.edgeFilter(edge_filter._jc) + else: + builder.edgeFilter(edge_filter) + + if checkpoint_interval > 0: + builder.setCheckpointInterval(checkpoint_interval) + + builder.setUseLocalCheckpoints(use_local_checkpoints) + builder.setIntermediateStorageLevel(storage_level_to_jvm(storage_level, self._spark)) + + jdf = builder.run() + return DataFrame(jdf, self._spark) + def aggregateMessages( self, aggCol: list[Column | str], diff --git a/python/graphframes/connect/graphframes_client.py b/python/graphframes/connect/graphframes_client.py index 88b769617..a56964ae2 100644 --- a/python/graphframes/connect/graphframes_client.py +++ b/python/graphframes/connect/graphframes_client.py @@ -546,6 +546,81 @@ def plan(self, session: SparkConnectClient) -> proto.Relation: self._spark, ) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, + ) -> DataFrame: + @final + class AllPaths(LogicalPlan): + def __init__( + self, + v: DataFrame, + e: DataFrame, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str, + max_path_length: int, + is_directed: bool, + checkpoint_interval: int, + use_local_checkpoints: bool, + storage_level: StorageLevel, + ) -> None: + super().__init__(None) + self.v = v + self.e = e + self.from_expr = from_expr + self.to_expr = to_expr + self.edge_filter = edge_filter + self.max_path_length = max_path_length + self.is_directed = is_directed + self.checkpoint_interval = checkpoint_interval + self.use_local_checkpoints = use_local_checkpoints + self.storage_level = storage_level + + @override + def plan(self, session: SparkConnectClient) -> proto.Relation: + graphframes_api_call = GraphFrameConnect._get_pb_api_message( + self.v, self.e, session + ) + graphframes_api_call.all_paths.CopyFrom( + pb.AllPaths( + from_expr=make_column_or_expr(self.from_expr, session), + to_expr=make_column_or_expr(self.to_expr, session), + edge_filter=make_column_or_expr(self.edge_filter, session), + max_path_length=self.max_path_length, + is_directed=self.is_directed, + checkpoint_interval=self.checkpoint_interval, + use_local_checkpoints=self.use_local_checkpoints, + storage_level=storage_level_to_proto(self.storage_level), + ) + ) + plan = self._create_proto_relation() + plan.extension.Pack(graphframes_api_call) + return plan + + if edge_filter is None: + edge_filter: Column = F.lit(True) + + return _dataframe_from_plan( + AllPaths( + v=self._vertices, + e=self._edges, + from_expr=from_expr, + to_expr=to_expr, + edge_filter=edge_filter, + max_path_length=max_path_length, + is_directed=is_directed, + ), + self._spark, + ) + def aggregateMessages( self, aggCol: list[Column | str], diff --git a/python/graphframes/connect/proto/graphframes_pb2.py b/python/graphframes/connect/proto/graphframes_pb2.py index 40fb37b15..ba5f54d87 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.py +++ b/python/graphframes/connect/proto/graphframes_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xed\x10\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighbors\x12n\n\x17neighborhood_aware_cdlp\x18\x19 \x01(\x0b\x32\x34.org.graphframes.connect.proto.NeighborhoodAwareCDLPH\x00R\x15neighborhoodAwareCdlpB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x88\x04\n\x15NeighborhoodAwareCDLP\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter\x12.\n\x13ignore_direct_links\x18\x02 \x01(\x08R\x11ignoreDirectLinks\x12H\n structural_similarity_multiplier\x18\x03 \x01(\x01R\x1estructuralSimilarityMultiplier\x12\x32\n\x15use_local_checkpoints\x18\x04 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x05 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x06 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x1f\n\x0bis_directed\x18\x07 \x01(\x08R\nisDirected\x12$\n\x0elg_nom_entries\x18\x08 \x01(\x05R\x0clgNomEntries\x12/\n\x11initial_label_col\x18\t \x01(\tH\x01R\x0finitialLabelCol\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x14\n\x12_initial_label_col"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' + b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xb5\x11\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighbors\x12n\n\x17neighborhood_aware_cdlp\x18\x19 \x01(\x0b\x32\x34.org.graphframes.connect.proto.NeighborhoodAwareCDLPH\x00R\x15neighborhoodAwareCdlp\x12\x46\n\tall_paths\x18\x1a \x01(\x0b\x32\'.org.graphframes.connect.proto.AllPathsH\x00R\x08\x61llPathsB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\x91\x04\n\x08\x41llPaths\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength\x12\x1f\n\x0bis_directed\x18\x05 \x01(\x08R\nisDirected\x12/\n\x13\x63heckpoint_interval\x18\x06 \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\x07 \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x08 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x88\x04\n\x15NeighborhoodAwareCDLP\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter\x12.\n\x13ignore_direct_links\x18\x02 \x01(\x08R\x11ignoreDirectLinks\x12H\n structural_similarity_multiplier\x18\x03 \x01(\x01R\x1estructuralSimilarityMultiplier\x12\x32\n\x15use_local_checkpoints\x18\x04 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x05 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x06 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x1f\n\x0bis_directed\x18\x07 \x01(\x08R\nisDirected\x12$\n\x0elg_nom_entries\x18\x08 \x01(\x05R\x0clgNomEntries\x12/\n\x11initial_label_col\x18\t \x01(\tH\x01R\x0finitialLabelCol\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x14\n\x12_initial_label_col"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' ) _globals = globals() @@ -31,57 +31,59 @@ "DESCRIPTOR" ]._serialized_options = b"\n!com.org.graphframes.connect.protoB\020GraphframesProtoH\001P\001\240\001\001\242\002\004OGCP\252\002\035Org.Graphframes.Connect.Proto\312\002\035Org\\Graphframes\\Connect\\Proto\342\002)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\352\002 Org::Graphframes::Connect::Proto" _globals["_GRAPHFRAMESAPI"]._serialized_start = 53 - _globals["_GRAPHFRAMESAPI"]._serialized_end = 2210 - _globals["_STORAGELEVEL"]._serialized_start = 2213 - _globals["_STORAGELEVEL"]._serialized_end = 2556 - _globals["_COLUMNOREXPRESSION"]._serialized_start = 2558 - _globals["_COLUMNOREXPRESSION"]._serialized_end = 2635 - _globals["_STRINGORLONGID"]._serialized_start = 2637 - _globals["_STRINGORLONGID"]._serialized_end = 2717 - _globals["_AGGREGATEMESSAGES"]._serialized_start = 2720 - _globals["_AGGREGATEMESSAGES"]._serialized_end = 3086 - _globals["_BFS"]._serialized_start = 3089 - _globals["_BFS"]._serialized_end = 3374 - _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3377 - _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 3767 - _globals["_DETECTINGCYCLES"]._serialized_start = 3770 - _globals["_DETECTINGCYCLES"]._serialized_end = 3993 - _globals["_DROPISOLATEDVERTICES"]._serialized_start = 3995 - _globals["_DROPISOLATEDVERTICES"]._serialized_end = 4017 - _globals["_FILTEREDGES"]._serialized_start = 4019 - _globals["_FILTEREDGES"]._serialized_end = 4113 - _globals["_FILTERVERTICES"]._serialized_start = 4115 - _globals["_FILTERVERTICES"]._serialized_end = 4212 - _globals["_FIND"]._serialized_start = 4214 - _globals["_FIND"]._serialized_end = 4246 - _globals["_LABELPROPAGATION"]._serialized_start = 4249 - _globals["_LABELPROPAGATION"]._serialized_end = 4530 - _globals["_NEIGHBORHOODAWARECDLP"]._serialized_start = 4533 - _globals["_NEIGHBORHOODAWARECDLP"]._serialized_end = 5053 - _globals["_PAGERANK"]._serialized_start = 5056 - _globals["_PAGERANK"]._serialized_end = 5282 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 5285 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 5465 - _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 5467 - _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 5585 - _globals["_PREGEL"]._serialized_start = 5588 - _globals["_PREGEL"]._serialized_end = 7002 - _globals["_SHORTESTPATHS"]._serialized_start = 7005 - _globals["_SHORTESTPATHS"]._serialized_end = 7387 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 7389 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 7445 - _globals["_SVDPLUSPLUS"]._serialized_start = 7448 - _globals["_SVDPLUSPLUS"]._serialized_end = 7662 - _globals["_TRIANGLECOUNT"]._serialized_start = 7665 - _globals["_TRIANGLECOUNT"]._serialized_end = 7896 - _globals["_TRIPLETS"]._serialized_start = 7898 - _globals["_TRIPLETS"]._serialized_end = 7908 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 7911 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 8160 - _globals["_KCORE"]._serialized_start = 8163 - _globals["_KCORE"]._serialized_end = 8376 - _globals["_AGGREGATENEIGHBORS"]._serialized_start = 8379 - _globals["_AGGREGATENEIGHBORS"]._serialized_end = 9475 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 9478 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 11015 + _globals["_GRAPHFRAMESAPI"]._serialized_end = 2282 + _globals["_STORAGELEVEL"]._serialized_start = 2285 + _globals["_STORAGELEVEL"]._serialized_end = 2628 + _globals["_COLUMNOREXPRESSION"]._serialized_start = 2630 + _globals["_COLUMNOREXPRESSION"]._serialized_end = 2707 + _globals["_STRINGORLONGID"]._serialized_start = 2709 + _globals["_STRINGORLONGID"]._serialized_end = 2789 + _globals["_AGGREGATEMESSAGES"]._serialized_start = 2792 + _globals["_AGGREGATEMESSAGES"]._serialized_end = 3158 + _globals["_BFS"]._serialized_start = 3161 + _globals["_BFS"]._serialized_end = 3446 + _globals["_ALLPATHS"]._serialized_start = 3449 + _globals["_ALLPATHS"]._serialized_end = 3978 + _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3981 + _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 4371 + _globals["_DETECTINGCYCLES"]._serialized_start = 4374 + _globals["_DETECTINGCYCLES"]._serialized_end = 4597 + _globals["_DROPISOLATEDVERTICES"]._serialized_start = 4599 + _globals["_DROPISOLATEDVERTICES"]._serialized_end = 4621 + _globals["_FILTEREDGES"]._serialized_start = 4623 + _globals["_FILTEREDGES"]._serialized_end = 4717 + _globals["_FILTERVERTICES"]._serialized_start = 4719 + _globals["_FILTERVERTICES"]._serialized_end = 4816 + _globals["_FIND"]._serialized_start = 4818 + _globals["_FIND"]._serialized_end = 4850 + _globals["_LABELPROPAGATION"]._serialized_start = 4853 + _globals["_LABELPROPAGATION"]._serialized_end = 5134 + _globals["_NEIGHBORHOODAWARECDLP"]._serialized_start = 5137 + _globals["_NEIGHBORHOODAWARECDLP"]._serialized_end = 5657 + _globals["_PAGERANK"]._serialized_start = 5660 + _globals["_PAGERANK"]._serialized_end = 5886 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 5889 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 6069 + _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 6071 + _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 6189 + _globals["_PREGEL"]._serialized_start = 6192 + _globals["_PREGEL"]._serialized_end = 7606 + _globals["_SHORTESTPATHS"]._serialized_start = 7609 + _globals["_SHORTESTPATHS"]._serialized_end = 7991 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 7993 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 8049 + _globals["_SVDPLUSPLUS"]._serialized_start = 8052 + _globals["_SVDPLUSPLUS"]._serialized_end = 8266 + _globals["_TRIANGLECOUNT"]._serialized_start = 8269 + _globals["_TRIANGLECOUNT"]._serialized_end = 8500 + _globals["_TRIPLETS"]._serialized_start = 8502 + _globals["_TRIPLETS"]._serialized_end = 8512 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 8515 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 8764 + _globals["_KCORE"]._serialized_start = 8767 + _globals["_KCORE"]._serialized_end = 8980 + _globals["_AGGREGATENEIGHBORS"]._serialized_start = 8983 + _globals["_AGGREGATENEIGHBORS"]._serialized_end = 10079 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 10082 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 11619 # @@protoc_insertion_point(module_scope) diff --git a/python/graphframes/connect/proto/graphframes_pb2.pyi b/python/graphframes/connect/proto/graphframes_pb2.pyi index 22a4f9f6b..d17140d5e 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.pyi +++ b/python/graphframes/connect/proto/graphframes_pb2.pyi @@ -37,6 +37,7 @@ class GraphFramesAPI(_message.Message): "rw_embeddings", "aggregate_neighbors", "neighborhood_aware_cdlp", + "all_paths", ) VERTICES_FIELD_NUMBER: _ClassVar[int] EDGES_FIELD_NUMBER: _ClassVar[int] @@ -63,6 +64,7 @@ class GraphFramesAPI(_message.Message): RW_EMBEDDINGS_FIELD_NUMBER: _ClassVar[int] AGGREGATE_NEIGHBORS_FIELD_NUMBER: _ClassVar[int] NEIGHBORHOOD_AWARE_CDLP_FIELD_NUMBER: _ClassVar[int] + ALL_PATHS_FIELD_NUMBER: _ClassVar[int] vertices: bytes edges: bytes aggregate_messages: AggregateMessages @@ -88,6 +90,7 @@ class GraphFramesAPI(_message.Message): rw_embeddings: RandomWalkEmbeddings aggregate_neighbors: AggregateNeighbors neighborhood_aware_cdlp: NeighborhoodAwareCDLP + all_paths: AllPaths def __init__( self, vertices: _Optional[bytes] = ..., @@ -119,6 +122,7 @@ class GraphFramesAPI(_message.Message): rw_embeddings: _Optional[_Union[RandomWalkEmbeddings, _Mapping]] = ..., aggregate_neighbors: _Optional[_Union[AggregateNeighbors, _Mapping]] = ..., neighborhood_aware_cdlp: _Optional[_Union[NeighborhoodAwareCDLP, _Mapping]] = ..., + all_paths: _Optional[_Union[AllPaths, _Mapping]] = ..., ) -> None: ... class StorageLevel(_message.Message): @@ -212,6 +216,45 @@ class BFS(_message.Message): max_path_length: _Optional[int] = ..., ) -> None: ... +class AllPaths(_message.Message): + __slots__ = ( + "from_expr", + "to_expr", + "edge_filter", + "max_path_length", + "is_directed", + "checkpoint_interval", + "use_local_checkpoints", + "storage_level", + ) + FROM_EXPR_FIELD_NUMBER: _ClassVar[int] + TO_EXPR_FIELD_NUMBER: _ClassVar[int] + EDGE_FILTER_FIELD_NUMBER: _ClassVar[int] + MAX_PATH_LENGTH_FIELD_NUMBER: _ClassVar[int] + IS_DIRECTED_FIELD_NUMBER: _ClassVar[int] + CHECKPOINT_INTERVAL_FIELD_NUMBER: _ClassVar[int] + USE_LOCAL_CHECKPOINTS_FIELD_NUMBER: _ClassVar[int] + STORAGE_LEVEL_FIELD_NUMBER: _ClassVar[int] + from_expr: ColumnOrExpression + to_expr: ColumnOrExpression + edge_filter: ColumnOrExpression + max_path_length: int + is_directed: bool + checkpoint_interval: int + use_local_checkpoints: bool + storage_level: StorageLevel + def __init__( + self, + from_expr: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + to_expr: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + edge_filter: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + max_path_length: _Optional[int] = ..., + is_directed: _Optional[bool] = ..., + checkpoint_interval: _Optional[int] = ..., + use_local_checkpoints: _Optional[bool] = ..., + storage_level: _Optional[_Union[StorageLevel, _Mapping]] = ..., + ) -> None: ... + class ConnectedComponents(_message.Message): __slots__ = ( "algorithm", diff --git a/python/graphframes/graphframe.py b/python/graphframes/graphframe.py index 7d7ac4695..81dc8bfca 100644 --- a/python/graphframes/graphframe.py +++ b/python/graphframes/graphframe.py @@ -501,6 +501,83 @@ def bfs( maxPathLength=maxPathLength, ) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, + ) -> DataFrame: + """ + Computes all simple paths between source and destination vertices. + + This algorithm enumerates paths up to ``max_path_length`` hops. It supports directed + and undirected traversal as well as optional edge filtering. It returns all simple + paths between source and destination vertices. Here the term "simple" means no + repeated vertices. For example, if there are paths A-B-C, A-D-C and the edge B-A, + and the user asked to find all the paths between "A" and "C", only A-B-C and A-D-C + will be returned, but not A-B-A-D-C. + + The default value of ``max_path_length`` is 5. Keep in mind that requesting + ``max_path_length`` on the scale of the graph diameter may cause the algorithm to + try to return (almost) all simple paths in the graph, which can create huge + performance degradation or even OOM-like errors. + + **Returned DataFrame schema:** + + - ``path``: array of vertex ids in traversal order + - ``len``: number of edges in the path (Long) + + .. note:: + In the case of an undirected graph, the algorithm runs on an internal graph + made by union of edges and reversed edges. It is assumed that the graph does + not have multi-edges. Results may be unstable and unpredictable for graphs + with multi-edges. + + **Example:** + + >>> paths = g.all_paths( + ... from_expr="name = 'A'", + ... to_expr="name = 'C'", + ... max_path_length=3, + ... ) + >>> paths.show() + + :param from_expr: Column expression or SQL expression string identifying the + source (starting) vertices. + :param to_expr: Column expression or SQL expression string identifying the + destination (target) vertices. + :param edge_filter: Optional Column expression or SQL expression string applied + to edges during traversal. Only edges satisfying this condition are considered. + If not provided, all edges are considered. + :param max_path_length: Maximum number of edges in a path; must be greater than 0. + Default is 5. Setting a large value (e.g., on the scale of the graph diameter) + may cause severe performance degradation or out-of-memory errors. + :param is_directed: Whether to use directed traversal. If False, the graph is + treated as undirected by internally unioning edges with reversed edges. + Default is True. + :param checkpoint_interval: Checkpoint every N iterations, 0 = disabled (default: 0) + :param use_local_checkpoints: Use local checkpoints (faster but less reliable) + :param storage_level: Storage level for intermediate results + + :return: DataFrame with columns ``path`` (array of vertex ids) and ``len`` + (number of edges in the path). + """ + return self._impl.all_paths( + from_expr=from_expr, + to_expr=to_expr, + edge_filter=edge_filter, + max_path_length=max_path_length, + is_directed=is_directed, + checkpoint_interval=checkpoint_interval, + use_local_checkpoints=use_local_checkpoints, + storage_level=storage_level, + ) + def aggregateMessages( self, aggCol: list[Column | str] | Column, diff --git a/python/tests/test_graphframes.py b/python/tests/test_graphframes.py index c235eba2a..d372cb903 100644 --- a/python/tests/test_graphframes.py +++ b/python/tests/test_graphframes.py @@ -351,6 +351,30 @@ def test_bfs(local_g: GraphFrame) -> None: assert paths3.count() == 0 +def test_all_paths(local_g: GraphFrame) -> None: + # local_g: A->B (love), B->A (hate), B->C (follow) + # Directed: A can reach C via A->B->C (1 path) + paths = local_g.all_paths("name='A'", "name='C'", use_local_checkpoints=True) + assert paths is not None + assert {"path", "len"}.issubset(set(paths.columns)) + paths_list = paths.collect() + assert len(paths_list) == 1 + assert paths_list[0]["len"] == 2 + + # With edge filter that removes the 'follow' edge: no path A->C + paths_filtered = local_g.all_paths("name='A'", "name='C'", edge_filter="action!='follow'", use_local_checkpoints=True) + assert paths_filtered.count() == 0 + + # Undirected: A->B->C and A->B->A (no simple path to C via A again), + # but also C->B->A is now possible, still only A->B->C from A to C + paths_undirected = local_g.all_paths("name='A'", "name='C'", is_directed=False, use_local_checkpoints=True) + assert paths_undirected.count() >= 1 + + # max_path_length too short: no paths + paths_short = local_g.all_paths("name='A'", "name='C'", max_path_length=1, use_local_checkpoints=True) + assert paths_short.count() == 0 + + def test_power_iteration_clustering(spark: SparkSession) -> None: vertices = [ (1, 0, 0.5),