From a83f620ad2f262a7286a408bcd75c922d6675790 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 29 Apr 2026 11:58:35 +0200 Subject: [PATCH 1/8] feat: all paths between two nodes --- .../scala/org/graphframes/GraphFrame.scala | 9 ++ .../scala/org/graphframes/lib/AllPaths.scala | 117 ++++++++++++++ .../org/graphframes/lib/AllPathsSuite.scala | 149 ++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 core/src/main/scala/org/graphframes/lib/AllPaths.scala create mode 100644 core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala diff --git a/core/src/main/scala/org/graphframes/GraphFrame.scala b/core/src/main/scala/org/graphframes/GraphFrame.scala index aa8eed9ef..c7bd20871 100644 --- a/core/src/main/scala/org/graphframes/GraphFrame.scala +++ b/core/src/main/scala/org/graphframes/GraphFrame.scala @@ -684,6 +684,15 @@ class GraphFrame private ( */ def bfs: BFS = new BFS(this) + /** + * Enumerate all paths between source and destination vertices. + * + * See [[org.graphframes.lib.AllPaths]] for details. + * + * @group stdlib + */ + def allPaths: AllPaths = new AllPaths(this) + /** * Aggregate information from neighboring vertices and edges through a controlled traversal. * diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala new file mode 100644 index 000000000..6bedd2eaa --- /dev/null +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.graphframes.lib + +import org.apache.spark.sql.Column +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.array +import org.apache.spark.sql.functions.array_contains +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.functions.concat +import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.functions.size +import org.apache.spark.sql.graphframes.SparkShims +import org.graphframes.GraphFrame +import org.graphframes.WithDirection + +/** + * Computes all simple paths between source and destination vertices. + * + * This algorithm enumerates paths up to `maxPathLength` hops. It supports directed and undirected + * traversal as well as optional edge filtering. + * + * Returned DataFrame schema: + * - `path`: array of vertex ids in traversal order + * - `len`: number of edges in the path (Long) + */ +class AllPaths private[graphframes] (private val graph: GraphFrame) + extends Arguments + with Serializable + with WithDirection { + + private var maxPathLength: Int = 10 + private var fromExpression: Column = _ + private var toExpression: Column = _ + private var edgeFilterExpression: Option[Column] = None + + def fromExpr(value: Column): this.type = { + fromExpression = value + this + } + + def fromExpr(value: String): this.type = fromExpr(expr(value)) + + def toExpr(value: Column): this.type = { + toExpression = value + this + } + + def toExpr(value: String): this.type = toExpr(expr(value)) + + def maxPathLength(value: Int): this.type = { + require(value > 0, s"AllPaths maxPathLength must be > 0, but was set to $value") + maxPathLength = value + this + } + + def edgeFilter(value: Column): this.type = { + edgeFilterExpression = Some(value) + this + } + + def edgeFilter(value: String): this.type = edgeFilter(expr(value)) + + def run(): DataFrame = { + require(fromExpression != null, "fromExpr is required.") + require(toExpression != null, "toExpr is required.") + + val traversalGraph = if (isDirected) { + graph + } else { + val edgeColumns = graph.edges.columns.toSeq + val reversed = graph.edges.select( + (Seq( + col(GraphFrame.DST).alias(GraphFrame.SRC), + col(GraphFrame.SRC).alias(GraphFrame.DST)) ++ + edgeColumns.filterNot(c => c == GraphFrame.SRC || c == GraphFrame.DST).map(col)): _*) + GraphFrame(graph.vertices, graph.edges.unionByName(reversed)) + } + + val agg = traversalGraph.aggregateNeighbors + .setStartingVertices(fromExpression) + .setMaxHops(maxPathLength) + .setTargetCondition(SparkShims.applyExprToCol(graph.spark, toExpression, "dst_attributes")) + .setStoppingCondition( + array_contains(col("path"), AggregateNeighbors.dstAttr(GraphFrame.ID))) + .addAccumulator( + "path", + array(col(GraphFrame.ID)), + concat(col("path"), array(AggregateNeighbors.dstAttr(GraphFrame.ID)))) + .setRequiredVertexAttributes(Seq(GraphFrame.ID)) + + edgeFilterExpression.foreach { ef => + agg.setEdgeFilter(SparkShims.applyExprToCol(graph.spark, ef, "edge_attributes")) + agg.setRequiredEdgeAttributes(graph.edges.columns.toSeq) + } + + agg + .run() + .select(col("path"), size(col("path")).cast("long").alias("len")) + .distinct() + } +} diff --git a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala new file mode 100644 index 000000000..c54a7d78b --- /dev/null +++ b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.graphframes.lib + +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.ArrayType +import org.apache.spark.sql.types.LongType +import org.graphframes.GraphFrame +import org.graphframes.GraphFrameTestSparkContext +import org.graphframes.SparkFunSuite + +class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { + + test("directed: enumerate all simple paths") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = + spark + .createDataFrame(Seq((1L, 2L, "x"), (2L, 4L, "x"), (1L, 3L, "x"), (3L, 4L, "x"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = + g.allPaths.fromExpr(col("id") === 1L).toExpr(col("id") === 4L).maxPathLength(3).run() + + val actual = result + .collect() + .map { row => + row.getAs[Seq[Long]]("path") -> row.getAs[Long]("len") + } + .toSet + + val expected = Set((Seq(1L, 2L, 4L), 3L), (Seq(1L, 3L, 4L), 3L)) + assert(actual === expected) + } + + test("undirected: traverse both edge directions") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = + spark + .createDataFrame(Seq((1L, 2L, "x"), (2L, 4L, "x"), (1L, 3L, "x"), (3L, 4L, "x"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths + .fromExpr(col("id") === 4L) + .toExpr(col("id") === 1L) + .setIsDirected(false) + .maxPathLength(3) + .run() + + val actual = result + .collect() + .map { row => + row.getAs[Seq[Long]]("path") -> row.getAs[Long]("len") + } + .toSet + + val expected = Set((Seq(4L, 2L, 1L), 3L), (Seq(4L, 3L, 1L), 3L)) + assert(actual === expected) + } + + test("edge filter excludes blocked edges") { + val vertices = + spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"), (4L, "D"))).toDF("id", "name") + val edges = spark + .createDataFrame( + Seq((1L, 2L, "allowed"), (2L, 4L, "allowed"), (1L, 3L, "blocked"), (3L, 4L, "allowed"))) + .toDF("src", "dst", "label") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths + .fromExpr("id = 1") + .toExpr("id = 4") + .edgeFilter(col("label") =!= "blocked") + .maxPathLength(3) + .run() + + val rows = result.collect() + assert(rows.length === 1) + assert(rows.head.getAs[Seq[Long]]("path") === Seq(1L, 2L, 4L)) + assert(rows.head.getAs[Long]("len") === 3L) + } + + test("cycle handling keeps paths simple") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"))).toDF("id", "name") + val edges = spark + .createDataFrame(Seq((1L, 2L), (2L, 1L), (2L, 3L), (1L, 3L))) + .toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths.fromExpr("id = 1").toExpr("id = 3").maxPathLength(4).run() + + val actualPaths = result.collect().map(_.getAs[Seq[Long]]("path")).toSet + assert(actualPaths === Set(Seq(1L, 3L), Seq(1L, 2L, 3L))) + assert(!actualPaths.contains(Seq(1L, 2L, 1L, 3L))) + } + + test("schema and required arguments") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"))).toDF("id", "name") + val edges = spark.createDataFrame(Seq((1L, 2L))).toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = g.allPaths.fromExpr("id = 1").toExpr("id = 2").run() + + assert(result.columns.toSeq === Seq("path", "len")) + assert(result.schema("path").dataType.asInstanceOf[ArrayType].elementType === LongType) + assert(result.schema("len").dataType === LongType) + + intercept[IllegalArgumentException] { + g.allPaths.run() + } + intercept[IllegalArgumentException] { + g.allPaths.fromExpr("id = 1").run() + } + intercept[IllegalArgumentException] { + g.allPaths.toExpr("id = 2").run() + } + intercept[IllegalArgumentException] { + g.allPaths.fromExpr("id = 1").toExpr("id = 2").maxPathLength(0) + } + } + + test("no matching path returns empty dataframe") { + val vertices = spark.createDataFrame(Seq((1L, "A"), (2L, "B"), (3L, "C"))).toDF("id", "name") + val edges = spark.createDataFrame(Seq((1L, 2L))).toDF("src", "dst") + val g = GraphFrame(vertices, edges) + + val result = + g.allPaths.fromExpr(col("id") === 1L).toExpr(col("id") === 3L).maxPathLength(4).run() + assert(result.collect().isEmpty) + } +} From fea94ca634d31054113a3de74d37809ad3dbe153 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 29 Apr 2026 12:59:50 +0200 Subject: [PATCH 2/8] fix: let's have some 2.12 vs 2.13 fun --- .../src/test/scala/org/graphframes/lib/AllPathsSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala index c54a7d78b..01cd3c2e5 100644 --- a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala +++ b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala @@ -107,9 +107,10 @@ class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { val result = g.allPaths.fromExpr("id = 1").toExpr("id = 3").maxPathLength(4).run() - val actualPaths = result.collect().map(_.getAs[Seq[Long]]("path")).toSet - assert(actualPaths === Set(Seq(1L, 3L), Seq(1L, 2L, 3L))) - assert(!actualPaths.contains(Seq(1L, 2L, 1L, 3L))) + val actualPaths = + result.collect().map(_.getAs[scala.collection.Seq[Long]]("path").toList).toSet + assert(actualPaths === Set(List(1L, 3L), List(1L, 2L, 3L))) + assert(!actualPaths.contains(List(1L, 2L, 1L, 3L))) } test("schema and required arguments") { From 06566fc803fddd7d12cab148ad6b318f1011ff99 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Mon, 1 Jun 2026 13:45:12 +0200 Subject: [PATCH 3/8] feat: adressing comments --- .../scala/org/graphframes/lib/AllPaths.scala | 22 ++++++++++++++----- .../org/graphframes/lib/AllPathsSuite.scala | 9 ++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala index 6bedd2eaa..3e9242d19 100644 --- a/core/src/main/scala/org/graphframes/lib/AllPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -24,7 +24,6 @@ import org.apache.spark.sql.functions.array_contains import org.apache.spark.sql.functions.col import org.apache.spark.sql.functions.concat import org.apache.spark.sql.functions.expr -import org.apache.spark.sql.functions.size import org.apache.spark.sql.graphframes.SparkShims import org.graphframes.GraphFrame import org.graphframes.WithDirection @@ -33,18 +32,29 @@ import org.graphframes.WithDirection * Computes all simple paths between source and destination vertices. * * This algorithm enumerates paths up to `maxPathLength` hops. It supports directed and undirected - * traversal as well as optional edge filtering. + * traversal as well as optional edge filtering. It returns all simple paths between source and + * destination vertices. Here the term "simple" means no repeated vertices. For example, if there + * are paths A-B-C, A-D-C and the edge B-A, user asked to find all the paths between "A" and "C" + * only A-B-C and A-D-C will be returned, but not the A-B-A-D-C. The default value of the + * `maxPathLength` is `5`. Keep in mind that requesting `maxPathLength` of the scale of the graph + * diameter may tend this algorithm will try to return (almost) all simple paths in the graph that + * can create huge performance degradation or even OOM-like errors. Algorithm supports both + * directed and undirected graphs. * * Returned DataFrame schema: * - `path`: array of vertex ids in traversal order * - `len`: number of edges in the path (Long) + * + * Note: in the case of undirected graph an algorithm run on the internal graph made by union + * edges and reversed edges. It is assummed that graph does not have multi-edges. Results may be + * unstable and unpredictable for the graph with multi-edges. */ class AllPaths private[graphframes] (private val graph: GraphFrame) extends Arguments with Serializable with WithDirection { - private var maxPathLength: Int = 10 + private var maxPathLength: Int = 5 private var fromExpression: Column = _ private var toExpression: Column = _ private var edgeFilterExpression: Option[Column] = None @@ -79,6 +89,9 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) def run(): DataFrame = { require(fromExpression != null, "fromExpr is required.") require(toExpression != null, "toExpr is required.") + require( + graph.vertices.columns.toSet.intersect(Set("hop", "path", "len")).isEmpty, + "columns `hop`, `path` and `len` are reserved by algorithm") val traversalGraph = if (isDirected) { graph @@ -102,7 +115,6 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) "path", array(col(GraphFrame.ID)), concat(col("path"), array(AggregateNeighbors.dstAttr(GraphFrame.ID)))) - .setRequiredVertexAttributes(Seq(GraphFrame.ID)) edgeFilterExpression.foreach { ef => agg.setEdgeFilter(SparkShims.applyExprToCol(graph.spark, ef, "edge_attributes")) @@ -111,7 +123,7 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) agg .run() - .select(col("path"), size(col("path")).cast("long").alias("len")) + .select(col("path"), col("hop").alias("len")) .distinct() } } diff --git a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala index 01cd3c2e5..4e3ef85a5 100644 --- a/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala +++ b/core/src/test/scala/org/graphframes/lib/AllPathsSuite.scala @@ -19,6 +19,7 @@ package org.graphframes.lib import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.ArrayType +import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.types.LongType import org.graphframes.GraphFrame import org.graphframes.GraphFrameTestSparkContext @@ -45,7 +46,7 @@ class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { } .toSet - val expected = Set((Seq(1L, 2L, 4L), 3L), (Seq(1L, 3L, 4L), 3L)) + val expected = Set((Seq(1L, 2L, 4L), 2L), (Seq(1L, 3L, 4L), 2L)) assert(actual === expected) } @@ -72,7 +73,7 @@ class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { } .toSet - val expected = Set((Seq(4L, 2L, 1L), 3L), (Seq(4L, 3L, 1L), 3L)) + val expected = Set((Seq(4L, 2L, 1L), 2L), (Seq(4L, 3L, 1L), 2L)) assert(actual === expected) } @@ -95,7 +96,7 @@ class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { val rows = result.collect() assert(rows.length === 1) assert(rows.head.getAs[Seq[Long]]("path") === Seq(1L, 2L, 4L)) - assert(rows.head.getAs[Long]("len") === 3L) + assert(rows.head.getAs[Long]("len") === 2L) } test("cycle handling keeps paths simple") { @@ -122,7 +123,7 @@ class AllPathsSuite extends SparkFunSuite with GraphFrameTestSparkContext { assert(result.columns.toSeq === Seq("path", "len")) assert(result.schema("path").dataType.asInstanceOf[ArrayType].elementType === LongType) - assert(result.schema("len").dataType === LongType) + assert(result.schema("len").dataType === IntegerType) intercept[IllegalArgumentException] { g.allPaths.run() From ea0243bf298560a5f505907d9c888a450b108ef3 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Mon, 1 Jun 2026 13:48:07 +0200 Subject: [PATCH 4/8] feat: addressing comments --- core/src/main/scala/org/graphframes/lib/AllPaths.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala index 3e9242d19..be0490ca9 100644 --- a/core/src/main/scala/org/graphframes/lib/AllPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -118,7 +118,6 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) edgeFilterExpression.foreach { ef => agg.setEdgeFilter(SparkShims.applyExprToCol(graph.spark, ef, "edge_attributes")) - agg.setRequiredEdgeAttributes(graph.edges.columns.toSeq) } agg From 7a27570175abf5d064f68c01eee835bd3edade46 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 10 Jun 2026 10:50:47 +0200 Subject: [PATCH 5/8] feat: PySpark bindings --- connect/src/main/protobuf/graphframes.proto | 9 ++ .../graphframes/GraphFramesConnectUtils.scala | 10 ++ .../scala/org/graphframes/lib/AllPaths.scala | 71 ++++++++++++ python/graphframes/classic/graphframe.py | 26 +++++ .../graphframes/connect/graphframes_client.py | 63 +++++++++++ .../connect/proto/graphframes_pb2.py | 106 +++++++++--------- .../connect/proto/graphframes_pb2.pyi | 25 +++++ python/graphframes/graphframe.py | 67 +++++++++++ python/tests/test_graphframes.py | 24 ++++ 9 files changed, 349 insertions(+), 52 deletions(-) diff --git a/connect/src/main/protobuf/graphframes.proto b/connect/src/main/protobuf/graphframes.proto index d29839dcb..8917eb7f7 100644 --- a/connect/src/main/protobuf/graphframes.proto +++ b/connect/src/main/protobuf/graphframes.proto @@ -39,6 +39,7 @@ message GraphFramesAPI { MaximalIndependentSet mis = 22; RandomWalkEmbeddings rw_embeddings = 23; AggregateNeighbors aggregate_neighbors = 24; + AllPaths all_paths = 25; } } @@ -87,6 +88,14 @@ message BFS { int32 max_path_length = 4; } +message AllPaths { + ColumnOrExpression from_expr = 1; + ColumnOrExpression to_expr = 2; + ColumnOrExpression edge_filter = 3; + int32 max_path_length = 4; + bool is_directed = 5; +} + message ConnectedComponents { string algorithm = 1; int32 checkpoint_interval = 2; diff --git a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala index 4c8b91a4e..95768b87d 100644 --- a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala +++ b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala @@ -202,6 +202,16 @@ object GraphFramesConnectUtils { .maxPathLength(bfsProto.getMaxPathLength) .run() } + case proto.GraphFramesAPI.MethodCase.ALL_PATHS => { + val allPathsProto = apiMessage.getAllPaths + graphFrame.allPaths + .toExpr(parseColumnOrExpression(allPathsProto.getToExpr, planner)) + .fromExpr(parseColumnOrExpression(allPathsProto.getFromExpr, planner)) + .edgeFilter(parseColumnOrExpression(allPathsProto.getEdgeFilter, planner)) + .maxPathLength(allPathsProto.getMaxPathLength) + .setIsDirected(allPathsProto.getIsDirected) + .run() + } case proto.GraphFramesAPI.MethodCase.CONNECTED_COMPONENTS => { val cc = apiMessage.getConnectedComponents val ccBuilder = graphFrame.connectedComponents diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala index be0490ca9..8c3b758e9 100644 --- a/core/src/main/scala/org/graphframes/lib/AllPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -59,33 +59,104 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) private var toExpression: Column = _ private var edgeFilterExpression: Option[Column] = None + /** + * Sets the expression identifying the source (starting) vertices. + * + * @param value + * a Column expression evaluated against vertex attributes to select source vertices + * @return + * this instance for method chaining + */ def fromExpr(value: Column): this.type = { fromExpression = value this } + /** + * Sets the expression identifying the source (starting) vertices. + * + * @param value + * a SQL expression string evaluated against vertex attributes to select source vertices + * @return + * this instance for method chaining + */ def fromExpr(value: String): this.type = fromExpr(expr(value)) + /** + * Sets the expression identifying the destination (target) vertices. + * + * @param value + * a Column expression evaluated against vertex attributes to select destination vertices + * @return + * this instance for method chaining + */ def toExpr(value: Column): this.type = { toExpression = value this } + /** + * Sets the expression identifying the destination (target) vertices. + * + * @param value + * a SQL expression string evaluated against vertex attributes to select destination vertices + * @return + * this instance for method chaining + */ def toExpr(value: String): this.type = toExpr(expr(value)) + /** + * Sets the maximum path length (number of edges) for the enumerated paths. + * + * Setting a large value (e.g. on the scale of the graph diameter) may cause the algorithm to + * attempt to collect a very large number of paths, leading to severe performance degradation or + * out-of-memory errors. Use with caution on large or densely connected graphs. + * + * @param value + * the maximum number of edges in a path; must be greater than 0. Default is 5. + * @return + * this instance for method chaining + */ def maxPathLength(value: Int): this.type = { require(value > 0, s"AllPaths maxPathLength must be > 0, but was set to $value") maxPathLength = value this } + /** + * Sets an optional filter expression applied to edges during traversal. Only edges satisfying + * this condition will be considered. + * + * @param value + * a Column expression evaluated against edge attributes + * @return + * this instance for method chaining + */ def edgeFilter(value: Column): this.type = { edgeFilterExpression = Some(value) this } + /** + * Sets an optional filter expression applied to edges during traversal. Only edges satisfying + * this condition will be considered. + * + * @param value + * a SQL expression string evaluated against edge attributes + * @return + * this instance for method chaining + */ def edgeFilter(value: String): this.type = edgeFilter(expr(value)) + /** + * Executes the AllPaths algorithm and returns all simple paths between the specified source and + * destination vertices. + * + * @return + * a DataFrame with the following columns: + * - `path`: an array of vertex ids in traversal order + * - `len`: the number of edges in the path (Long) + */ def run(): DataFrame = { require(fromExpression != null, "fromExpr is required.") require(toExpression != null, "toExpr is required.") diff --git a/python/graphframes/classic/graphframe.py b/python/graphframes/classic/graphframe.py index e20cd0572..15aed7d3a 100644 --- a/python/graphframes/classic/graphframe.py +++ b/python/graphframes/classic/graphframe.py @@ -140,6 +140,32 @@ def bfs( jdf = builder.run() return DataFrame(jdf, self._spark) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + ) -> DataFrame: + builder = self._jvm_graph.allPaths() + if isinstance(from_expr, Column): + builder = builder.fromExpr(from_expr._jc) + else: + builder = builder.fromExpr(from_expr) + if isinstance(to_expr, Column): + builder = builder.toExpr(to_expr._jc) + else: + builder = builder.toExpr(to_expr) + builder = builder.maxPathLength(max_path_length).setIsDirected(is_directed) + if edge_filter is not None: + if isinstance(edge_filter, Column): + builder = builder.edgeFilter(edge_filter._jc) + else: + builder = builder.edgeFilter(edge_filter) + jdf = builder.run() + return DataFrame(jdf, self._spark) + def aggregateMessages( self, aggCol: list[Column | str], diff --git a/python/graphframes/connect/graphframes_client.py b/python/graphframes/connect/graphframes_client.py index ad76aa2e1..557a49c97 100644 --- a/python/graphframes/connect/graphframes_client.py +++ b/python/graphframes/connect/graphframes_client.py @@ -546,6 +546,69 @@ def plan(self, session: SparkConnectClient) -> proto.Relation: self._spark, ) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + ) -> DataFrame: + @final + class AllPaths(LogicalPlan): + def __init__( + self, + v: DataFrame, + e: DataFrame, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str, + max_path_length: int, + is_directed: bool, + ) -> None: + super().__init__(None) + self.v = v + self.e = e + self.from_expr = from_expr + self.to_expr = to_expr + self.edge_filter = edge_filter + self.max_path_length = max_path_length + self.is_directed = is_directed + + @override + def plan(self, session: SparkConnectClient) -> proto.Relation: + graphframes_api_call = GraphFrameConnect._get_pb_api_message( + self.v, self.e, session + ) + graphframes_api_call.all_paths.CopyFrom( + pb.AllPaths( + from_expr=make_column_or_expr(self.from_expr, session), + to_expr=make_column_or_expr(self.to_expr, session), + edge_filter=make_column_or_expr(self.edge_filter, session), + max_path_length=self.max_path_length, + is_directed=self.is_directed, + ) + ) + plan = self._create_proto_relation() + plan.extension.Pack(graphframes_api_call) + return plan + + if edge_filter is None: + edge_filter: Column = F.lit(True) + + return _dataframe_from_plan( + AllPaths( + v=self._vertices, + e=self._edges, + from_expr=from_expr, + to_expr=to_expr, + edge_filter=edge_filter, + max_path_length=max_path_length, + is_directed=is_directed, + ), + self._spark, + ) + def aggregateMessages( self, aggCol: list[Column | str], diff --git a/python/graphframes/connect/proto/graphframes_pb2.py b/python/graphframes/connect/proto/graphframes_pb2.py index a24c08400..175512172 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.py +++ b/python/graphframes/connect/proto/graphframes_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xfd\x0f\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighborsB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' + b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xc5\x10\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighbors\x12\x46\n\tall_paths\x18\x19 \x01(\x0b\x32\'.org.graphframes.connect.proto.AllPathsH\x00R\x08\x61llPathsB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\xc3\x02\n\x08\x41llPaths\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength\x12\x1f\n\x0bis_directed\x18\x05 \x01(\x08R\nisDirected"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' ) _globals = globals() @@ -31,55 +31,57 @@ "DESCRIPTOR" ]._serialized_options = b"\n!com.org.graphframes.connect.protoB\020GraphframesProtoH\001P\001\240\001\001\242\002\004OGCP\252\002\035Org.Graphframes.Connect.Proto\312\002\035Org\\Graphframes\\Connect\\Proto\342\002)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\352\002 Org::Graphframes::Connect::Proto" _globals["_GRAPHFRAMESAPI"]._serialized_start = 53 - _globals["_GRAPHFRAMESAPI"]._serialized_end = 2098 - _globals["_STORAGELEVEL"]._serialized_start = 2101 - _globals["_STORAGELEVEL"]._serialized_end = 2444 - _globals["_COLUMNOREXPRESSION"]._serialized_start = 2446 - _globals["_COLUMNOREXPRESSION"]._serialized_end = 2523 - _globals["_STRINGORLONGID"]._serialized_start = 2525 - _globals["_STRINGORLONGID"]._serialized_end = 2605 - _globals["_AGGREGATEMESSAGES"]._serialized_start = 2608 - _globals["_AGGREGATEMESSAGES"]._serialized_end = 2974 - _globals["_BFS"]._serialized_start = 2977 - _globals["_BFS"]._serialized_end = 3262 - _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3265 - _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 3655 - _globals["_DETECTINGCYCLES"]._serialized_start = 3658 - _globals["_DETECTINGCYCLES"]._serialized_end = 3881 - _globals["_DROPISOLATEDVERTICES"]._serialized_start = 3883 - _globals["_DROPISOLATEDVERTICES"]._serialized_end = 3905 - _globals["_FILTEREDGES"]._serialized_start = 3907 - _globals["_FILTEREDGES"]._serialized_end = 4001 - _globals["_FILTERVERTICES"]._serialized_start = 4003 - _globals["_FILTERVERTICES"]._serialized_end = 4100 - _globals["_FIND"]._serialized_start = 4102 - _globals["_FIND"]._serialized_end = 4134 - _globals["_LABELPROPAGATION"]._serialized_start = 4137 - _globals["_LABELPROPAGATION"]._serialized_end = 4418 - _globals["_PAGERANK"]._serialized_start = 4421 - _globals["_PAGERANK"]._serialized_end = 4647 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 4650 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 4830 - _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 4832 - _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 4950 - _globals["_PREGEL"]._serialized_start = 4953 - _globals["_PREGEL"]._serialized_end = 6367 - _globals["_SHORTESTPATHS"]._serialized_start = 6370 - _globals["_SHORTESTPATHS"]._serialized_end = 6752 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 6754 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 6810 - _globals["_SVDPLUSPLUS"]._serialized_start = 6813 - _globals["_SVDPLUSPLUS"]._serialized_end = 7027 - _globals["_TRIANGLECOUNT"]._serialized_start = 7030 - _globals["_TRIANGLECOUNT"]._serialized_end = 7261 - _globals["_TRIPLETS"]._serialized_start = 7263 - _globals["_TRIPLETS"]._serialized_end = 7273 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 7276 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 7525 - _globals["_KCORE"]._serialized_start = 7528 - _globals["_KCORE"]._serialized_end = 7741 - _globals["_AGGREGATENEIGHBORS"]._serialized_start = 7744 - _globals["_AGGREGATENEIGHBORS"]._serialized_end = 8840 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 8843 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 10380 + _globals["_GRAPHFRAMESAPI"]._serialized_end = 2170 + _globals["_STORAGELEVEL"]._serialized_start = 2173 + _globals["_STORAGELEVEL"]._serialized_end = 2516 + _globals["_COLUMNOREXPRESSION"]._serialized_start = 2518 + _globals["_COLUMNOREXPRESSION"]._serialized_end = 2595 + _globals["_STRINGORLONGID"]._serialized_start = 2597 + _globals["_STRINGORLONGID"]._serialized_end = 2677 + _globals["_AGGREGATEMESSAGES"]._serialized_start = 2680 + _globals["_AGGREGATEMESSAGES"]._serialized_end = 3046 + _globals["_BFS"]._serialized_start = 3049 + _globals["_BFS"]._serialized_end = 3334 + _globals["_ALLPATHS"]._serialized_start = 3337 + _globals["_ALLPATHS"]._serialized_end = 3660 + _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3663 + _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 4053 + _globals["_DETECTINGCYCLES"]._serialized_start = 4056 + _globals["_DETECTINGCYCLES"]._serialized_end = 4279 + _globals["_DROPISOLATEDVERTICES"]._serialized_start = 4281 + _globals["_DROPISOLATEDVERTICES"]._serialized_end = 4303 + _globals["_FILTEREDGES"]._serialized_start = 4305 + _globals["_FILTEREDGES"]._serialized_end = 4399 + _globals["_FILTERVERTICES"]._serialized_start = 4401 + _globals["_FILTERVERTICES"]._serialized_end = 4498 + _globals["_FIND"]._serialized_start = 4500 + _globals["_FIND"]._serialized_end = 4532 + _globals["_LABELPROPAGATION"]._serialized_start = 4535 + _globals["_LABELPROPAGATION"]._serialized_end = 4816 + _globals["_PAGERANK"]._serialized_start = 4819 + _globals["_PAGERANK"]._serialized_end = 5045 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 5048 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 5228 + _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 5230 + _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 5348 + _globals["_PREGEL"]._serialized_start = 5351 + _globals["_PREGEL"]._serialized_end = 6765 + _globals["_SHORTESTPATHS"]._serialized_start = 6768 + _globals["_SHORTESTPATHS"]._serialized_end = 7150 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 7152 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 7208 + _globals["_SVDPLUSPLUS"]._serialized_start = 7211 + _globals["_SVDPLUSPLUS"]._serialized_end = 7425 + _globals["_TRIANGLECOUNT"]._serialized_start = 7428 + _globals["_TRIANGLECOUNT"]._serialized_end = 7659 + _globals["_TRIPLETS"]._serialized_start = 7661 + _globals["_TRIPLETS"]._serialized_end = 7671 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 7674 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 7923 + _globals["_KCORE"]._serialized_start = 7926 + _globals["_KCORE"]._serialized_end = 8139 + _globals["_AGGREGATENEIGHBORS"]._serialized_start = 8142 + _globals["_AGGREGATENEIGHBORS"]._serialized_end = 9238 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 9241 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 10778 # @@protoc_insertion_point(module_scope) diff --git a/python/graphframes/connect/proto/graphframes_pb2.pyi b/python/graphframes/connect/proto/graphframes_pb2.pyi index 86d2edc1f..78f30b35a 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.pyi +++ b/python/graphframes/connect/proto/graphframes_pb2.pyi @@ -36,6 +36,7 @@ class GraphFramesAPI(_message.Message): "mis", "rw_embeddings", "aggregate_neighbors", + "all_paths", ) VERTICES_FIELD_NUMBER: _ClassVar[int] EDGES_FIELD_NUMBER: _ClassVar[int] @@ -61,6 +62,7 @@ class GraphFramesAPI(_message.Message): MIS_FIELD_NUMBER: _ClassVar[int] RW_EMBEDDINGS_FIELD_NUMBER: _ClassVar[int] AGGREGATE_NEIGHBORS_FIELD_NUMBER: _ClassVar[int] + ALL_PATHS_FIELD_NUMBER: _ClassVar[int] vertices: bytes edges: bytes aggregate_messages: AggregateMessages @@ -85,6 +87,7 @@ class GraphFramesAPI(_message.Message): mis: MaximalIndependentSet rw_embeddings: RandomWalkEmbeddings aggregate_neighbors: AggregateNeighbors + all_paths: AllPaths def __init__( self, vertices: _Optional[bytes] = ..., @@ -115,6 +118,7 @@ class GraphFramesAPI(_message.Message): mis: _Optional[_Union[MaximalIndependentSet, _Mapping]] = ..., rw_embeddings: _Optional[_Union[RandomWalkEmbeddings, _Mapping]] = ..., aggregate_neighbors: _Optional[_Union[AggregateNeighbors, _Mapping]] = ..., + all_paths: _Optional[_Union[AllPaths, _Mapping]] = ..., ) -> None: ... class StorageLevel(_message.Message): @@ -208,6 +212,27 @@ class BFS(_message.Message): max_path_length: _Optional[int] = ..., ) -> None: ... +class AllPaths(_message.Message): + __slots__ = ("from_expr", "to_expr", "edge_filter", "max_path_length", "is_directed") + FROM_EXPR_FIELD_NUMBER: _ClassVar[int] + TO_EXPR_FIELD_NUMBER: _ClassVar[int] + EDGE_FILTER_FIELD_NUMBER: _ClassVar[int] + MAX_PATH_LENGTH_FIELD_NUMBER: _ClassVar[int] + IS_DIRECTED_FIELD_NUMBER: _ClassVar[int] + from_expr: ColumnOrExpression + to_expr: ColumnOrExpression + edge_filter: ColumnOrExpression + max_path_length: int + is_directed: bool + def __init__( + self, + from_expr: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + to_expr: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + edge_filter: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., + max_path_length: _Optional[int] = ..., + is_directed: _Optional[bool] = ..., + ) -> None: ... + class ConnectedComponents(_message.Message): __slots__ = ( "algorithm", diff --git a/python/graphframes/graphframe.py b/python/graphframes/graphframe.py index 7791b85ca..e57681d60 100644 --- a/python/graphframes/graphframe.py +++ b/python/graphframes/graphframe.py @@ -501,6 +501,73 @@ def bfs( maxPathLength=maxPathLength, ) + def all_paths( + self, + from_expr: Column | str, + to_expr: Column | str, + edge_filter: Column | str | None = None, + max_path_length: int = 5, + is_directed: bool = True, + ) -> DataFrame: + """ + Computes all simple paths between source and destination vertices. + + This algorithm enumerates paths up to ``max_path_length`` hops. It supports directed + and undirected traversal as well as optional edge filtering. It returns all simple + paths between source and destination vertices. Here the term "simple" means no + repeated vertices. For example, if there are paths A-B-C, A-D-C and the edge B-A, + and the user asked to find all the paths between "A" and "C", only A-B-C and A-D-C + will be returned, but not A-B-A-D-C. + + The default value of ``max_path_length`` is 5. Keep in mind that requesting + ``max_path_length`` on the scale of the graph diameter may cause the algorithm to + try to return (almost) all simple paths in the graph, which can create huge + performance degradation or even OOM-like errors. + + **Returned DataFrame schema:** + + - ``path``: array of vertex ids in traversal order + - ``len``: number of edges in the path (Long) + + .. note:: + In the case of an undirected graph, the algorithm runs on an internal graph + made by union of edges and reversed edges. It is assumed that the graph does + not have multi-edges. Results may be unstable and unpredictable for graphs + with multi-edges. + + **Example:** + + >>> paths = g.all_paths( + ... from_expr="name = 'A'", + ... to_expr="name = 'C'", + ... max_path_length=3, + ... ) + >>> paths.show() + + :param from_expr: Column expression or SQL expression string identifying the + source (starting) vertices. + :param to_expr: Column expression or SQL expression string identifying the + destination (target) vertices. + :param edge_filter: Optional Column expression or SQL expression string applied + to edges during traversal. Only edges satisfying this condition are considered. + If not provided, all edges are considered. + :param max_path_length: Maximum number of edges in a path; must be greater than 0. + Default is 5. Setting a large value (e.g., on the scale of the graph diameter) + may cause severe performance degradation or out-of-memory errors. + :param is_directed: Whether to use directed traversal. If False, the graph is + treated as undirected by internally unioning edges with reversed edges. + Default is True. + :return: DataFrame with columns ``path`` (array of vertex ids) and ``len`` + (number of edges in the path). + """ + return self._impl.all_paths( + from_expr=from_expr, + to_expr=to_expr, + edge_filter=edge_filter, + max_path_length=max_path_length, + is_directed=is_directed, + ) + def aggregateMessages( self, aggCol: list[Column | str] | Column, diff --git a/python/tests/test_graphframes.py b/python/tests/test_graphframes.py index 3e2ccfb7f..8868fcb17 100644 --- a/python/tests/test_graphframes.py +++ b/python/tests/test_graphframes.py @@ -351,6 +351,30 @@ def test_bfs(local_g: GraphFrame) -> None: assert paths3.count() == 0 +def test_all_paths(local_g: GraphFrame) -> None: + # local_g: A->B (love), B->A (hate), B->C (follow) + # Directed: A can reach C via A->B->C (1 path) + paths = local_g.all_paths("name='A'", "name='C'") + assert paths is not None + assert {"path", "len"}.issubset(set(paths.columns)) + paths_list = paths.collect() + assert len(paths_list) == 1 + assert paths_list[0]["len"] == 2 + + # With edge filter that removes the 'follow' edge: no path A->C + paths_filtered = local_g.all_paths("name='A'", "name='C'", edge_filter="action!='follow'") + assert paths_filtered.count() == 0 + + # Undirected: A->B->C and A->B->A (no simple path to C via A again), + # but also C->B->A is now possible, still only A->B->C from A to C + paths_undirected = local_g.all_paths("name='A'", "name='C'", is_directed=False) + assert paths_undirected.count() >= 1 + + # max_path_length too short: no paths + paths_short = local_g.all_paths("name='A'", "name='C'", max_path_length=1) + assert paths_short.count() == 0 + + def test_power_iteration_clustering(spark: SparkSession) -> None: vertices = [ (1, 0, 0.5), From 52f6315533e82506d52658443787f7c363e442d1 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 10 Jun 2026 11:15:53 +0200 Subject: [PATCH 6/8] docs: add docs page --- docs/src/04-user-guide/05-traversals.md | 101 +++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/docs/src/04-user-guide/05-traversals.md b/docs/src/04-user-guide/05-traversals.md index 9c8bc6d62..479b2e212 100644 --- a/docs/src/04-user-guide/05-traversals.md +++ b/docs/src/04-user-guide/05-traversals.md @@ -67,6 +67,104 @@ The level of storage for intermediate results and the output `DataFrame` with co By default this is true and algorithm will look for only directed paths. By passing false, graph will be considered as undirected and algorithm will look for any shortest path. +## All paths + +Computes all simple paths between source and destination vertices up to a specified maximum path length. A "simple" path means no repeated vertices — for example, if there are paths A-B-C and A-D-C and an edge B-A, querying for all paths between A and C returns A-B-C and A-D-C but not A-B-A-D-C. + +The algorithm supports both directed and undirected traversal as well as optional edge filtering. + +--- + +**WARNING:** + +_Depending on the value of `max_path_length`, this algorithm can be **extremely slow** or even lead to **out-of-memory (OOM) errors** or **huge disk spills**. Setting `max_path_length` on the scale of the graph diameter causes the algorithm to attempt to enumerate (almost) all simple paths in the graph, which grows combinatorially. Use with caution, especially on large or densely connected graphs. Start with a small value and increase gradually._ + +--- + +### Python API + +For API details, refer to the @:pydoc(graphframes.GraphFrame.all_paths). + +```python +from graphframes.examples import Graphs + +g = Graphs(spark).friends() # Get example graph + +# Find all simple paths from vertex "a" to vertex "d" with at most 3 edges +paths = g.all_paths( + from_expr="id = 'a'", + to_expr="id = 'd'", + max_path_length=3, +) +paths.select("path", "len").show() + +# Search with edge filter and undirected traversal +paths = g.all_paths( + from_expr="id = 'a'", + to_expr="id = 'd'", + edge_filter="relationship != 'friend'", + max_path_length=3, + is_directed=False, +) +paths.select("path", "len").show() +``` + +### Scala API + +For API details, refer to the @:scaladoc(org.graphframes.lib.AllPaths). + +```scala +import org.graphframes.{examples, GraphFrame} + +val g: GraphFrame = examples.Graphs.friends // get example graph + +// Find all simple paths from vertex "a" to vertex "d" with at most 3 edges +val paths = g.allPaths + .fromExpr("id = 'a'") + .toExpr("id = 'd'") + .maxPathLength(3) + .run() +paths.select("path", "len").show() + +// Search with edge filter +val pathsFiltered = g.allPaths + .fromExpr("id = 'a'") + .toExpr("id = 'd'") + .edgeFilter("relationship != 'friend'") + .maxPathLength(3) + .run() +pathsFiltered.select("path", "len").show() +``` + +### Returned DataFrame Schema + +The returned `DataFrame` contains the following columns: + +- `path`: an array of vertex IDs in traversal order (e.g., `[a, b, d]`) +- `len`: the number of edges in the path (Long) + +### Arguments + +- `from_expr` (Python) / `fromExpr` (Scala) + +A Column or SQL expression identifying the source (starting) vertices. + +- `to_expr` (Python) / `toExpr` (Scala) + +A Column or SQL expression identifying the destination (target) vertices. + +- `max_path_length` (Python) / `maxPathLength` (Scala) + +Maximum number of edges in a path; must be greater than 0. Default is 5. Setting a large value (e.g., on the scale of the graph diameter) may cause severe performance degradation or out-of-memory errors. + +- `edge_filter` (Python) / `edgeFilter` (Scala) + +An optional Column or SQL expression applied to edges during traversal. Only edges satisfying this condition are considered. + +- `is_directed` (Python) / directed mode (Scala) + +Whether to use directed traversal. Default is `True`. If `False`, the graph is treated as undirected by internally unioning edges with reversed edges. Note: it is assumed that the graph does not have multi-edges. Results may be unstable for graphs with multi-edges. + ## Breadth-first search (BFS) Breadth-first search (BFS) finds the shortest path(s) from one vertex (or a set of vertices) to another vertex (or a set @@ -455,6 +553,7 @@ This algorithm is particularly useful for: - **Accumulative Computations**: Computing values that depend on the entire path (e.g., product of edge weights, sum of node values) Unlike single-hop algorithms like BFS or shortest paths, Aggregate Neighbors allows you to: + - Track multiple accumulators simultaneously - Define custom stopping criteria - Access vertex and edge attributes during traversal @@ -464,7 +563,7 @@ Unlike single-hop algorithms like BFS or shortest paths, Aggregate Neighbors all **NOTE** -*Be aware, that returned `DataFrame` is persistent and should be unpersisted manually after processing to avoid memory leaks!* +_Be aware, that returned `DataFrame` is persistent and should be unpersisted manually after processing to avoid memory leaks!_ --- From bc6906770fd7871dc76de0fd49d5ca8dcecbac75 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 10 Jun 2026 13:56:35 +0200 Subject: [PATCH 7/8] feat: propagate all the agg_nbrs args --- connect/src/main/protobuf/graphframes.proto | 3 + .../graphframes/GraphFramesConnectUtils.scala | 1 + .../scala/org/graphframes/lib/AllPaths.scala | 11 ++- python/graphframes/classic/graphframe.py | 24 +++-- .../graphframes/connect/graphframes_client.py | 12 +++ .../connect/proto/graphframes_pb2.py | 88 +++++++++---------- .../connect/proto/graphframes_pb2.pyi | 20 ++++- python/graphframes/graphframe.py | 10 +++ 8 files changed, 116 insertions(+), 53 deletions(-) diff --git a/connect/src/main/protobuf/graphframes.proto b/connect/src/main/protobuf/graphframes.proto index da9850d0a..489f2585b 100644 --- a/connect/src/main/protobuf/graphframes.proto +++ b/connect/src/main/protobuf/graphframes.proto @@ -95,6 +95,9 @@ message AllPaths { ColumnOrExpression edge_filter = 3; int32 max_path_length = 4; bool is_directed = 5; + int32 checkpoint_interval = 6; + bool use_local_checkpoints = 7; + optional StorageLevel storage_level = 8; } message ConnectedComponents { diff --git a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala index 3c97ca2a5..887022048 100644 --- a/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala +++ b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala @@ -210,6 +210,7 @@ object GraphFramesConnectUtils { .edgeFilter(parseColumnOrExpression(allPathsProto.getEdgeFilter, planner)) .maxPathLength(allPathsProto.getMaxPathLength) .setIsDirected(allPathsProto.getIsDirected) + .setUseLocalCheckpoints(allPathsProto.getUseLocalCheckpoints) .run() } case proto.GraphFramesAPI.MethodCase.CONNECTED_COMPONENTS => { diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala index 8c3b758e9..279b8d01a 100644 --- a/core/src/main/scala/org/graphframes/lib/AllPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.functions.concat import org.apache.spark.sql.functions.expr import org.apache.spark.sql.graphframes.SparkShims import org.graphframes.GraphFrame +import org.graphframes.WithCheckpointInterval import org.graphframes.WithDirection +import org.graphframes.WithIntermediateStorageLevel +import org.graphframes.WithLocalCheckpoints /** * Computes all simple paths between source and destination vertices. @@ -52,7 +55,10 @@ import org.graphframes.WithDirection class AllPaths private[graphframes] (private val graph: GraphFrame) extends Arguments with Serializable - with WithDirection { + with WithDirection + with WithLocalCheckpoints + with WithCheckpointInterval + with WithIntermediateStorageLevel { private var maxPathLength: Int = 5 private var fromExpression: Column = _ @@ -186,6 +192,9 @@ class AllPaths private[graphframes] (private val graph: GraphFrame) "path", array(col(GraphFrame.ID)), concat(col("path"), array(AggregateNeighbors.dstAttr(GraphFrame.ID)))) + .setUseLocalCheckpoints(useLocalCheckpoints) + .setCheckpointInterval(checkpointInterval) + .setIntermediateStorageLevel(intermediateStorageLevel) edgeFilterExpression.foreach { ef => agg.setEdgeFilter(SparkShims.applyExprToCol(graph.spark, ef, "edge_attributes")) diff --git a/python/graphframes/classic/graphframe.py b/python/graphframes/classic/graphframe.py index 37e490bed..fe0ffdbbc 100644 --- a/python/graphframes/classic/graphframe.py +++ b/python/graphframes/classic/graphframe.py @@ -147,22 +147,32 @@ def all_paths( edge_filter: Column | str | None = None, max_path_length: int = 5, is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, ) -> DataFrame: builder = self._jvm_graph.allPaths() if isinstance(from_expr, Column): - builder = builder.fromExpr(from_expr._jc) + builder.fromExpr(from_expr._jc) else: - builder = builder.fromExpr(from_expr) + builder.fromExpr(from_expr) if isinstance(to_expr, Column): - builder = builder.toExpr(to_expr._jc) + builder.toExpr(to_expr._jc) else: - builder = builder.toExpr(to_expr) - builder = builder.maxPathLength(max_path_length).setIsDirected(is_directed) + builder.toExpr(to_expr) + builder.maxPathLength(max_path_length).setIsDirected(is_directed) if edge_filter is not None: if isinstance(edge_filter, Column): - builder = builder.edgeFilter(edge_filter._jc) + builder.edgeFilter(edge_filter._jc) else: - builder = builder.edgeFilter(edge_filter) + builder.edgeFilter(edge_filter) + + if checkpoint_interval > 0: + builder.setCheckpointInterval(checkpoint_interval) + + builder.setUseLocalCheckpoints(use_local_checkpoints) + builder.setIntermediateStorageLevel(storage_level_to_jvm(storage_level, self._spark)) + jdf = builder.run() return DataFrame(jdf, self._spark) diff --git a/python/graphframes/connect/graphframes_client.py b/python/graphframes/connect/graphframes_client.py index a2d8f41fd..a56964ae2 100644 --- a/python/graphframes/connect/graphframes_client.py +++ b/python/graphframes/connect/graphframes_client.py @@ -553,6 +553,9 @@ def all_paths( edge_filter: Column | str | None = None, max_path_length: int = 5, is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, ) -> DataFrame: @final class AllPaths(LogicalPlan): @@ -565,6 +568,9 @@ def __init__( edge_filter: Column | str, max_path_length: int, is_directed: bool, + checkpoint_interval: int, + use_local_checkpoints: bool, + storage_level: StorageLevel, ) -> None: super().__init__(None) self.v = v @@ -574,6 +580,9 @@ def __init__( self.edge_filter = edge_filter self.max_path_length = max_path_length self.is_directed = is_directed + self.checkpoint_interval = checkpoint_interval + self.use_local_checkpoints = use_local_checkpoints + self.storage_level = storage_level @override def plan(self, session: SparkConnectClient) -> proto.Relation: @@ -587,6 +596,9 @@ def plan(self, session: SparkConnectClient) -> proto.Relation: edge_filter=make_column_or_expr(self.edge_filter, session), max_path_length=self.max_path_length, is_directed=self.is_directed, + checkpoint_interval=self.checkpoint_interval, + use_local_checkpoints=self.use_local_checkpoints, + storage_level=storage_level_to_proto(self.storage_level), ) ) plan = self._create_proto_relation() diff --git a/python/graphframes/connect/proto/graphframes_pb2.py b/python/graphframes/connect/proto/graphframes_pb2.py index 2813cd88e..ba5f54d87 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.py +++ b/python/graphframes/connect/proto/graphframes_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xb5\x11\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighbors\x12n\n\x17neighborhood_aware_cdlp\x18\x19 \x01(\x0b\x32\x34.org.graphframes.connect.proto.NeighborhoodAwareCDLPH\x00R\x15neighborhoodAwareCdlp\x12\x46\n\tall_paths\x18\x1a \x01(\x0b\x32\'.org.graphframes.connect.proto.AllPathsH\x00R\x08\x61llPathsB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\xc3\x02\n\x08\x41llPaths\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength\x12\x1f\n\x0bis_directed\x18\x05 \x01(\x08R\nisDirected"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x88\x04\n\x15NeighborhoodAwareCDLP\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter\x12.\n\x13ignore_direct_links\x18\x02 \x01(\x08R\x11ignoreDirectLinks\x12H\n structural_similarity_multiplier\x18\x03 \x01(\x01R\x1estructuralSimilarityMultiplier\x12\x32\n\x15use_local_checkpoints\x18\x04 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x05 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x06 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x1f\n\x0bis_directed\x18\x07 \x01(\x08R\nisDirected\x12$\n\x0elg_nom_entries\x18\x08 \x01(\x05R\x0clgNomEntries\x12/\n\x11initial_label_col\x18\t \x01(\tH\x01R\x0finitialLabelCol\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x14\n\x12_initial_label_col"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' + b'\n\x11graphframes.proto\x12\x1dorg.graphframes.connect.proto"\xb5\x11\n\x0eGraphFramesAPI\x12\x1a\n\x08vertices\x18\x01 \x01(\x0cR\x08vertices\x12\x14\n\x05\x65\x64ges\x18\x02 \x01(\x0cR\x05\x65\x64ges\x12\x61\n\x12\x61ggregate_messages\x18\x03 \x01(\x0b\x32\x30.org.graphframes.connect.proto.AggregateMessagesH\x00R\x11\x61ggregateMessages\x12\x36\n\x03\x62\x66s\x18\x04 \x01(\x0b\x32".org.graphframes.connect.proto.BFSH\x00R\x03\x62\x66s\x12g\n\x14\x63onnected_components\x18\x05 \x01(\x0b\x32\x32.org.graphframes.connect.proto.ConnectedComponentsH\x00R\x13\x63onnectedComponents\x12k\n\x16\x64rop_isolated_vertices\x18\x06 \x01(\x0b\x32\x33.org.graphframes.connect.proto.DropIsolatedVerticesH\x00R\x14\x64ropIsolatedVertices\x12[\n\x10\x64\x65tecting_cycles\x18\x07 \x01(\x0b\x32..org.graphframes.connect.proto.DetectingCyclesH\x00R\x0f\x64\x65tectingCycles\x12O\n\x0c\x66ilter_edges\x18\x08 \x01(\x0b\x32*.org.graphframes.connect.proto.FilterEdgesH\x00R\x0b\x66ilterEdges\x12X\n\x0f\x66ilter_vertices\x18\t \x01(\x0b\x32-.org.graphframes.connect.proto.FilterVerticesH\x00R\x0e\x66ilterVertices\x12\x39\n\x04\x66ind\x18\n \x01(\x0b\x32#.org.graphframes.connect.proto.FindH\x00R\x04\x66ind\x12^\n\x11label_propagation\x18\x0b \x01(\x0b\x32/.org.graphframes.connect.proto.LabelPropagationH\x00R\x10labelPropagation\x12\x46\n\tpage_rank\x18\x0c \x01(\x0b\x32\'.org.graphframes.connect.proto.PageRankH\x00R\x08pageRank\x12\x84\x01\n\x1fparallel_personalized_page_rank\x18\r \x01(\x0b\x32;.org.graphframes.connect.proto.ParallelPersonalizedPageRankH\x00R\x1cparallelPersonalizedPageRank\x12w\n\x1apower_iteration_clustering\x18\x0e \x01(\x0b\x32\x37.org.graphframes.connect.proto.PowerIterationClusteringH\x00R\x18powerIterationClustering\x12?\n\x06pregel\x18\x0f \x01(\x0b\x32%.org.graphframes.connect.proto.PregelH\x00R\x06pregel\x12U\n\x0eshortest_paths\x18\x10 \x01(\x0b\x32,.org.graphframes.connect.proto.ShortestPathsH\x00R\rshortestPaths\x12\x80\x01\n\x1dstrongly_connected_components\x18\x11 \x01(\x0b\x32:.org.graphframes.connect.proto.StronglyConnectedComponentsH\x00R\x1bstronglyConnectedComponents\x12P\n\rsvd_plus_plus\x18\x12 \x01(\x0b\x32*.org.graphframes.connect.proto.SVDPlusPlusH\x00R\x0bsvdPlusPlus\x12U\n\x0etriangle_count\x18\x13 \x01(\x0b\x32,.org.graphframes.connect.proto.TriangleCountH\x00R\rtriangleCount\x12\x45\n\x08triplets\x18\x14 \x01(\x0b\x32\'.org.graphframes.connect.proto.TripletsH\x00R\x08triplets\x12<\n\x05kcore\x18\x15 \x01(\x0b\x32$.org.graphframes.connect.proto.KCoreH\x00R\x05kcore\x12H\n\x03mis\x18\x16 \x01(\x0b\x32\x34.org.graphframes.connect.proto.MaximalIndependentSetH\x00R\x03mis\x12Z\n\rrw_embeddings\x18\x17 \x01(\x0b\x32\x33.org.graphframes.connect.proto.RandomWalkEmbeddingsH\x00R\x0crwEmbeddings\x12\x64\n\x13\x61ggregate_neighbors\x18\x18 \x01(\x0b\x32\x31.org.graphframes.connect.proto.AggregateNeighborsH\x00R\x12\x61ggregateNeighbors\x12n\n\x17neighborhood_aware_cdlp\x18\x19 \x01(\x0b\x32\x34.org.graphframes.connect.proto.NeighborhoodAwareCDLPH\x00R\x15neighborhoodAwareCdlp\x12\x46\n\tall_paths\x18\x1a \x01(\x0b\x32\'.org.graphframes.connect.proto.AllPathsH\x00R\x08\x61llPathsB\x08\n\x06method"\xd7\x02\n\x0cStorageLevel\x12\x1d\n\tdisk_only\x18\x01 \x01(\x08H\x00R\x08\x64iskOnly\x12 \n\x0b\x64isk_only_2\x18\x02 \x01(\x08H\x00R\tdiskOnly2\x12 \n\x0b\x64isk_only_3\x18\x03 \x01(\x08H\x00R\tdiskOnly3\x12(\n\x0fmemory_and_disk\x18\x04 \x01(\x08H\x00R\rmemoryAndDisk\x12+\n\x11memory_and_disk_2\x18\x05 \x01(\x08H\x00R\x0ememoryAndDisk2\x12\x33\n\x15memory_and_disk_deser\x18\x06 \x01(\x08H\x00R\x12memoryAndDiskDeser\x12!\n\x0bmemory_only\x18\x07 \x01(\x08H\x00R\nmemoryOnly\x12$\n\rmemory_only_2\x18\x08 \x01(\x08H\x00R\x0bmemoryOnly2B\x0f\n\rstorage_level"M\n\x12\x43olumnOrExpression\x12\x12\n\x03\x63ol\x18\x01 \x01(\x0cH\x00R\x03\x63ol\x12\x14\n\x04\x65xpr\x18\x02 \x01(\tH\x00R\x04\x65xprB\r\n\x0b\x63ol_or_expr"P\n\x0eStringOrLongID\x12\x19\n\x07long_id\x18\x01 \x01(\x03H\x00R\x06longId\x12\x1d\n\tstring_id\x18\x02 \x01(\tH\x00R\x08stringIdB\x04\n\x02id"\xee\x02\n\x11\x41ggregateMessages\x12J\n\x07\x61gg_col\x18\x01 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06\x61ggCol\x12Q\n\x0bsend_to_src\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToSrc\x12Q\n\x0bsend_to_dst\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tsendToDst\x12U\n\rstorage_level\x18\x04 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x9d\x02\n\x03\x42\x46S\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength"\x91\x04\n\x08\x41llPaths\x12N\n\tfrom_expr\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x08\x66romExpr\x12J\n\x07to_expr\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x06toExpr\x12R\n\x0b\x65\x64ge_filter\x18\x03 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\nedgeFilter\x12&\n\x0fmax_path_length\x18\x04 \x01(\x05R\rmaxPathLength\x12\x1f\n\x0bis_directed\x18\x05 \x01(\x08R\nisDirected\x12/\n\x13\x63heckpoint_interval\x18\x06 \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\x07 \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x08 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x86\x03\n\x13\x43onnectedComponents\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12/\n\x13\x62roadcast_threshold\x18\x03 \x01(\x05R\x12\x62roadcastThreshold\x12\x37\n\x18use_labels_as_components\x18\x04 \x01(\x08R\x15useLabelsAsComponents\x12\x32\n\x15use_local_checkpoints\x18\x05 \x01(\x08R\x13useLocalCheckpoints\x12\x19\n\x08max_iter\x18\x06 \x01(\x05R\x07maxIter\x12U\n\rstorage_level\x18\x07 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xdf\x01\n\x0f\x44\x65tectingCycles\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x16\n\x14\x44ropIsolatedVertices"^\n\x0b\x46ilterEdges\x12O\n\tcondition\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition"a\n\x0e\x46ilterVertices\x12O\n\tcondition\x18\x02 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\tcondition" \n\x04\x46ind\x12\x18\n\x07pattern\x18\x01 \x01(\tR\x07pattern"\x99\x02\n\x10LabelPropagation\x12\x1c\n\talgorithm\x18\x01 \x01(\tR\talgorithm\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\x88\x04\n\x15NeighborhoodAwareCDLP\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter\x12.\n\x13ignore_direct_links\x18\x02 \x01(\x08R\x11ignoreDirectLinks\x12H\n structural_similarity_multiplier\x18\x03 \x01(\x01R\x1estructuralSimilarityMultiplier\x12\x32\n\x15use_local_checkpoints\x18\x04 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x05 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x06 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x1f\n\x0bis_directed\x18\x07 \x01(\x08R\nisDirected\x12$\n\x0elg_nom_entries\x18\x08 \x01(\x05R\x0clgNomEntries\x12/\n\x11initial_label_col\x18\t \x01(\tH\x01R\x0finitialLabelCol\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x14\n\x12_initial_label_col"\xe2\x01\n\x08PageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12O\n\tsource_id\x18\x02 \x01(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDH\x00R\x08sourceId\x88\x01\x01\x12\x1e\n\x08max_iter\x18\x03 \x01(\x05H\x01R\x07maxIter\x88\x01\x01\x12\x15\n\x03tol\x18\x04 \x01(\x01H\x02R\x03tol\x88\x01\x01\x42\x0c\n\n_source_idB\x0b\n\t_max_iterB\x06\n\x04_tol"\xb4\x01\n\x1cParallelPersonalizedPageRank\x12+\n\x11reset_probability\x18\x01 \x01(\x01R\x10resetProbability\x12L\n\nsource_ids\x18\x02 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tsourceIds\x12\x19\n\x08max_iter\x18\x03 \x01(\x05R\x07maxIter"v\n\x18PowerIterationClustering\x12\x0c\n\x01k\x18\x01 \x01(\x05R\x01k\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12"\n\nweight_col\x18\x03 \x01(\tH\x00R\tweightCol\x88\x01\x01\x42\r\n\x0b_weight_col"\x86\x0b\n\x06Pregel\x12L\n\x08\x61gg_msgs\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x07\x61ggMsgs\x12X\n\x0fsend_msg_to_dst\x18\x02 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToDst\x12X\n\x0fsend_msg_to_src\x18\x03 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x0csendMsgToSrc\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12\x19\n\x08max_iter\x18\x05 \x01(\x05R\x07maxIter\x12.\n\x13\x61\x64\x64itional_col_name\x18\x06 \x01(\tR\x11\x61\x64\x64itionalColName\x12g\n\x16\x61\x64\x64itional_col_initial\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x14\x61\x64\x64itionalColInitial\x12_\n\x12\x61\x64\x64itional_col_upd\x18\x08 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x64\x64itionalColUpd\x12*\n\x0e\x65\x61rly_stopping\x18\t \x01(\x08H\x00R\rearlyStopping\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\n \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0b \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x01R\x0cstorageLevel\x88\x01\x01\x12\x37\n\x16stop_if_all_non_active\x18\x0c \x01(\x08H\x02R\x12stopIfAllNonActive\x88\x01\x01\x12\x66\n\x13initial_active_expr\x18\r \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x03R\x11initialActiveExpr\x88\x01\x01\x12\x64\n\x12update_active_expr\x18\x0e \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x04R\x10updateActiveExpr\x88\x01\x01\x12\x45\n\x1dskip_messages_from_non_active\x18\x0f \x01(\x08H\x05R\x19skipMessagesFromNonActive\x88\x01\x01\x12\x35\n\x14required_src_columns\x18\x10 \x01(\tH\x06R\x12requiredSrcColumns\x88\x01\x01\x12\x35\n\x14required_dst_columns\x18\x11 \x01(\tH\x07R\x12requiredDstColumns\x88\x01\x01\x42\x11\n\x0f_early_stoppingB\x10\n\x0e_storage_levelB\x19\n\x17_stop_if_all_non_activeB\x16\n\x14_initial_active_exprB\x15\n\x13_update_active_exprB \n\x1e_skip_messages_from_non_activeB\x17\n\x15_required_src_columnsB\x17\n\x15_required_dst_columns"\xfe\x02\n\rShortestPaths\x12K\n\tlandmarks\x18\x01 \x03(\x0b\x32-.org.graphframes.connect.proto.StringOrLongIDR\tlandmarks\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x04 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x05 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12$\n\x0bis_directed\x18\x06 \x01(\x08H\x01R\nisDirected\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0e\n\x0c_is_directed"8\n\x1bStronglyConnectedComponents\x12\x19\n\x08max_iter\x18\x01 \x01(\x05R\x07maxIter"\xd6\x01\n\x0bSVDPlusPlus\x12\x12\n\x04rank\x18\x01 \x01(\x05R\x04rank\x12\x19\n\x08max_iter\x18\x02 \x01(\x05R\x07maxIter\x12\x1b\n\tmin_value\x18\x03 \x01(\x01R\x08minValue\x12\x1b\n\tmax_value\x18\x04 \x01(\x01R\x08maxValue\x12\x16\n\x06gamma1\x18\x05 \x01(\x01R\x06gamma1\x12\x16\n\x06gamma2\x18\x06 \x01(\x01R\x06gamma2\x12\x16\n\x06gamma6\x18\x07 \x01(\x01R\x06gamma6\x12\x16\n\x06gamma7\x18\x08 \x01(\x01R\x06gamma7"\xe7\x01\n\rTriangleCount\x12U\n\rstorage_level\x18\x01 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12!\n\talgorithm\x18\x02 \x01(\tH\x01R\talgorithm\x88\x01\x01\x12)\n\x0elg_nom_entries\x18\x03 \x01(\x05H\x02R\x0clgNomEntries\x88\x01\x01\x42\x10\n\x0e_storage_levelB\x0c\n\n_algorithmB\x11\n\x0f_lg_nom_entries"\n\n\x08Triplets"\xf9\x01\n\x15MaximalIndependentSet\x12/\n\x13\x63heckpoint_interval\x18\x01 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x02 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x12\x32\n\x15use_local_checkpoints\x18\x03 \x01(\x08R\x13useLocalCheckpoints\x12\x12\n\x04seed\x18\x04 \x01(\x03R\x04seedB\x10\n\x0e_storage_level"\xd5\x01\n\x05KCore\x12\x32\n\x15use_local_checkpoints\x18\x01 \x01(\x08R\x13useLocalCheckpoints\x12/\n\x13\x63heckpoint_interval\x18\x02 \x01(\x05R\x12\x63heckpointInterval\x12U\n\rstorage_level\x18\x03 \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xc8\x08\n\x12\x41ggregateNeighbors\x12^\n\x11starting_vertices\x18\x01 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10startingVertices\x12\x19\n\x08max_hops\x18\x02 \x01(\x05R\x07maxHops\x12+\n\x11\x61\x63\x63umulator_names\x18\x03 \x03(\tR\x10\x61\x63\x63umulatorNames\x12^\n\x11\x61\x63\x63umulator_inits\x18\x04 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x10\x61\x63\x63umulatorInits\x12\x62\n\x13\x61\x63\x63umulator_updates\x18\x05 \x03(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionR\x12\x61\x63\x63umulatorUpdates\x12\x65\n\x12stopping_condition\x18\x06 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x00R\x11stoppingCondition\x88\x01\x01\x12\x61\n\x10target_condition\x18\x07 \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x01R\x0ftargetCondition\x88\x01\x01\x12<\n\x1arequired_vertex_attributes\x18\x08 \x03(\tR\x18requiredVertexAttributes\x12\x38\n\x18required_edge_attributes\x18\t \x03(\tR\x16requiredEdgeAttributes\x12W\n\x0b\x65\x64ge_filter\x18\n \x01(\x0b\x32\x31.org.graphframes.connect.proto.ColumnOrExpressionH\x02R\nedgeFilter\x88\x01\x01\x12!\n\x0cremove_loops\x18\x0b \x01(\x08R\x0bremoveLoops\x12/\n\x13\x63heckpoint_interval\x18\x0c \x01(\x05R\x12\x63heckpointInterval\x12\x32\n\x15use_local_checkpoints\x18\r \x01(\x08R\x13useLocalCheckpoints\x12U\n\rstorage_level\x18\x0e \x01(\x0b\x32+.org.graphframes.connect.proto.StorageLevelH\x03R\x0cstorageLevel\x88\x01\x01\x42\x15\n\x13_stopping_conditionB\x13\n\x11_target_conditionB\x0e\n\x0c_edge_filterB\x10\n\x0e_storage_level"\x81\x0c\n\x14RandomWalkEmbeddings\x12,\n\x12use_edge_direction\x18\x01 \x01(\x08R\x10useEdgeDirection\x12\x19\n\x08rw_model\x18\x02 \x01(\tR\x07rwModel\x12\x1e\n\x0brw_max_nbrs\x18\x03 \x01(\x05R\trwMaxNbrs\x12\x30\n\x15rw_num_walks_per_node\x18\x04 \x01(\x05R\x11rwNumWalksPerNode\x12"\n\rrw_batch_size\x18\x05 \x01(\x05R\x0brwBatchSize\x12$\n\x0erw_num_batches\x18\x06 \x01(\x05R\x0crwNumBatches\x12\x17\n\x07rw_seed\x18\x07 \x01(\x03R\x06rwSeed\x12\x34\n\x16rw_restart_probability\x18\x08 \x01(\x01R\x14rwRestartProbability\x12.\n\x13rw_temporary_prefix\x18\t \x01(\tR\x11rwTemporaryPrefix\x12&\n\x0frw_cached_walks\x18\n \x01(\tR\rrwCachedWalks\x12%\n\x0esequence_model\x18\x0b \x01(\tR\rsequenceModel\x12\x32\n\x15hash2vec_context_size\x18\x0c \x01(\x05R\x13hash2vecContextSize\x12\x36\n\x17hash2vec_num_partitions\x18\r \x01(\x05R\x15hash2vecNumPartitions\x12\x36\n\x17hash2vec_embeddings_dim\x18\x0e \x01(\x05R\x15hash2vecEmbeddingsDim\x12\x36\n\x17hash2vec_decay_function\x18\x0f \x01(\tR\x15hash2vecDecayFunction\x12\x36\n\x17hash2vec_gaussian_sigma\x18\x10 \x01(\x01R\x15hash2vecGaussianSigma\x12\x32\n\x15hash2vec_hashing_seed\x18\x11 \x01(\x05R\x13hash2vecHashingSeed\x12,\n\x12hash2vec_sign_seed\x18\x12 \x01(\x05R\x10hash2vecSignSeed\x12-\n\x13hash2vec_do_l2_norm\x18\x13 \x01(\x08R\x10hash2vecDoL2Norm\x12(\n\x10hash2vec_safe_l2\x18\x14 \x01(\x08R\x0ehash2vecSafeL2\x12*\n\x11word2vec_max_iter\x18\x15 \x01(\x05R\x0fword2vecMaxIter\x12\x36\n\x17word2vec_embeddings_dim\x18\x16 \x01(\x05R\x15word2vecEmbeddingsDim\x12\x30\n\x14word2vec_window_size\x18\x17 \x01(\x05R\x12word2vecWindowSize\x12\x36\n\x17word2vec_num_partitions\x18\x18 \x01(\x05R\x15word2vecNumPartitions\x12,\n\x12word2vec_min_count\x18\x19 \x01(\x05R\x10word2vecMinCount\x12?\n\x1cword2vec_max_sentence_length\x18\x1a \x01(\x05R\x19word2vecMaxSentenceLength\x12#\n\rword2vec_seed\x18\x1b \x01(\x03R\x0cword2vecSeed\x12,\n\x12word2vec_step_size\x18\x1c \x01(\x01R\x10word2vecStepSize\x12/\n\x13\x61ggregate_neighbors\x18\x1d \x01(\x08R\x12\x61ggregateNeighbors\x12?\n\x1c\x61ggregate_neighbors_max_nbrs\x18\x1e \x01(\x05R\x19\x61ggregateNeighborsMaxNbrs\x12\x38\n\x18\x61ggregate_neighbors_seed\x18\x1f \x01(\x03R\x16\x61ggregateNeighborsSeed\x12+\n\x12\x63lean_up_after_run\x18 \x01(\x08R\x0f\x63leanUpAfterRunB\xd2\x01\n!com.org.graphframes.connect.protoB\x10GraphframesProtoH\x01P\x01\xa0\x01\x01\xa2\x02\x04OGCP\xaa\x02\x1dOrg.Graphframes.Connect.Proto\xca\x02\x1dOrg\\Graphframes\\Connect\\Proto\xe2\x02)Org\\Graphframes\\Connect\\Proto\\GPBMetadata\xea\x02 Org::Graphframes::Connect::Protob\x06proto3' ) _globals = globals() @@ -43,47 +43,47 @@ _globals["_BFS"]._serialized_start = 3161 _globals["_BFS"]._serialized_end = 3446 _globals["_ALLPATHS"]._serialized_start = 3449 - _globals["_ALLPATHS"]._serialized_end = 3772 - _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3775 - _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 4165 - _globals["_DETECTINGCYCLES"]._serialized_start = 4168 - _globals["_DETECTINGCYCLES"]._serialized_end = 4391 - _globals["_DROPISOLATEDVERTICES"]._serialized_start = 4393 - _globals["_DROPISOLATEDVERTICES"]._serialized_end = 4415 - _globals["_FILTEREDGES"]._serialized_start = 4417 - _globals["_FILTEREDGES"]._serialized_end = 4511 - _globals["_FILTERVERTICES"]._serialized_start = 4513 - _globals["_FILTERVERTICES"]._serialized_end = 4610 - _globals["_FIND"]._serialized_start = 4612 - _globals["_FIND"]._serialized_end = 4644 - _globals["_LABELPROPAGATION"]._serialized_start = 4647 - _globals["_LABELPROPAGATION"]._serialized_end = 4928 - _globals["_NEIGHBORHOODAWARECDLP"]._serialized_start = 4931 - _globals["_NEIGHBORHOODAWARECDLP"]._serialized_end = 5451 - _globals["_PAGERANK"]._serialized_start = 5454 - _globals["_PAGERANK"]._serialized_end = 5680 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 5683 - _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 5863 - _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 5865 - _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 5983 - _globals["_PREGEL"]._serialized_start = 5986 - _globals["_PREGEL"]._serialized_end = 7400 - _globals["_SHORTESTPATHS"]._serialized_start = 7403 - _globals["_SHORTESTPATHS"]._serialized_end = 7785 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 7787 - _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 7843 - _globals["_SVDPLUSPLUS"]._serialized_start = 7846 - _globals["_SVDPLUSPLUS"]._serialized_end = 8060 - _globals["_TRIANGLECOUNT"]._serialized_start = 8063 - _globals["_TRIANGLECOUNT"]._serialized_end = 8294 - _globals["_TRIPLETS"]._serialized_start = 8296 - _globals["_TRIPLETS"]._serialized_end = 8306 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 8309 - _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 8558 - _globals["_KCORE"]._serialized_start = 8561 - _globals["_KCORE"]._serialized_end = 8774 - _globals["_AGGREGATENEIGHBORS"]._serialized_start = 8777 - _globals["_AGGREGATENEIGHBORS"]._serialized_end = 9873 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 9876 - _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 11413 + _globals["_ALLPATHS"]._serialized_end = 3978 + _globals["_CONNECTEDCOMPONENTS"]._serialized_start = 3981 + _globals["_CONNECTEDCOMPONENTS"]._serialized_end = 4371 + _globals["_DETECTINGCYCLES"]._serialized_start = 4374 + _globals["_DETECTINGCYCLES"]._serialized_end = 4597 + _globals["_DROPISOLATEDVERTICES"]._serialized_start = 4599 + _globals["_DROPISOLATEDVERTICES"]._serialized_end = 4621 + _globals["_FILTEREDGES"]._serialized_start = 4623 + _globals["_FILTEREDGES"]._serialized_end = 4717 + _globals["_FILTERVERTICES"]._serialized_start = 4719 + _globals["_FILTERVERTICES"]._serialized_end = 4816 + _globals["_FIND"]._serialized_start = 4818 + _globals["_FIND"]._serialized_end = 4850 + _globals["_LABELPROPAGATION"]._serialized_start = 4853 + _globals["_LABELPROPAGATION"]._serialized_end = 5134 + _globals["_NEIGHBORHOODAWARECDLP"]._serialized_start = 5137 + _globals["_NEIGHBORHOODAWARECDLP"]._serialized_end = 5657 + _globals["_PAGERANK"]._serialized_start = 5660 + _globals["_PAGERANK"]._serialized_end = 5886 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_start = 5889 + _globals["_PARALLELPERSONALIZEDPAGERANK"]._serialized_end = 6069 + _globals["_POWERITERATIONCLUSTERING"]._serialized_start = 6071 + _globals["_POWERITERATIONCLUSTERING"]._serialized_end = 6189 + _globals["_PREGEL"]._serialized_start = 6192 + _globals["_PREGEL"]._serialized_end = 7606 + _globals["_SHORTESTPATHS"]._serialized_start = 7609 + _globals["_SHORTESTPATHS"]._serialized_end = 7991 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_start = 7993 + _globals["_STRONGLYCONNECTEDCOMPONENTS"]._serialized_end = 8049 + _globals["_SVDPLUSPLUS"]._serialized_start = 8052 + _globals["_SVDPLUSPLUS"]._serialized_end = 8266 + _globals["_TRIANGLECOUNT"]._serialized_start = 8269 + _globals["_TRIANGLECOUNT"]._serialized_end = 8500 + _globals["_TRIPLETS"]._serialized_start = 8502 + _globals["_TRIPLETS"]._serialized_end = 8512 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_start = 8515 + _globals["_MAXIMALINDEPENDENTSET"]._serialized_end = 8764 + _globals["_KCORE"]._serialized_start = 8767 + _globals["_KCORE"]._serialized_end = 8980 + _globals["_AGGREGATENEIGHBORS"]._serialized_start = 8983 + _globals["_AGGREGATENEIGHBORS"]._serialized_end = 10079 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_start = 10082 + _globals["_RANDOMWALKEMBEDDINGS"]._serialized_end = 11619 # @@protoc_insertion_point(module_scope) diff --git a/python/graphframes/connect/proto/graphframes_pb2.pyi b/python/graphframes/connect/proto/graphframes_pb2.pyi index 2e64f7de9..d17140d5e 100644 --- a/python/graphframes/connect/proto/graphframes_pb2.pyi +++ b/python/graphframes/connect/proto/graphframes_pb2.pyi @@ -217,17 +217,32 @@ class BFS(_message.Message): ) -> None: ... class AllPaths(_message.Message): - __slots__ = ("from_expr", "to_expr", "edge_filter", "max_path_length", "is_directed") + __slots__ = ( + "from_expr", + "to_expr", + "edge_filter", + "max_path_length", + "is_directed", + "checkpoint_interval", + "use_local_checkpoints", + "storage_level", + ) FROM_EXPR_FIELD_NUMBER: _ClassVar[int] TO_EXPR_FIELD_NUMBER: _ClassVar[int] EDGE_FILTER_FIELD_NUMBER: _ClassVar[int] MAX_PATH_LENGTH_FIELD_NUMBER: _ClassVar[int] IS_DIRECTED_FIELD_NUMBER: _ClassVar[int] + CHECKPOINT_INTERVAL_FIELD_NUMBER: _ClassVar[int] + USE_LOCAL_CHECKPOINTS_FIELD_NUMBER: _ClassVar[int] + STORAGE_LEVEL_FIELD_NUMBER: _ClassVar[int] from_expr: ColumnOrExpression to_expr: ColumnOrExpression edge_filter: ColumnOrExpression max_path_length: int is_directed: bool + checkpoint_interval: int + use_local_checkpoints: bool + storage_level: StorageLevel def __init__( self, from_expr: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., @@ -235,6 +250,9 @@ class AllPaths(_message.Message): edge_filter: _Optional[_Union[ColumnOrExpression, _Mapping]] = ..., max_path_length: _Optional[int] = ..., is_directed: _Optional[bool] = ..., + checkpoint_interval: _Optional[int] = ..., + use_local_checkpoints: _Optional[bool] = ..., + storage_level: _Optional[_Union[StorageLevel, _Mapping]] = ..., ) -> None: ... class ConnectedComponents(_message.Message): diff --git a/python/graphframes/graphframe.py b/python/graphframes/graphframe.py index 9636f2590..81dc8bfca 100644 --- a/python/graphframes/graphframe.py +++ b/python/graphframes/graphframe.py @@ -508,6 +508,9 @@ def all_paths( edge_filter: Column | str | None = None, max_path_length: int = 5, is_directed: bool = True, + checkpoint_interval: int = 2, + use_local_checkpoints: bool = False, + storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, ) -> DataFrame: """ Computes all simple paths between source and destination vertices. @@ -557,6 +560,10 @@ def all_paths( :param is_directed: Whether to use directed traversal. If False, the graph is treated as undirected by internally unioning edges with reversed edges. Default is True. + :param checkpoint_interval: Checkpoint every N iterations, 0 = disabled (default: 0) + :param use_local_checkpoints: Use local checkpoints (faster but less reliable) + :param storage_level: Storage level for intermediate results + :return: DataFrame with columns ``path`` (array of vertex ids) and ``len`` (number of edges in the path). """ @@ -566,6 +573,9 @@ def all_paths( edge_filter=edge_filter, max_path_length=max_path_length, is_directed=is_directed, + checkpoint_interval=checkpoint_interval, + use_local_checkpoints=use_local_checkpoints, + storage_level=storage_level, ) def aggregateMessages( From 41b8bc33290f4ba24088127dc646e6e2e4a1ec52 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 10 Jun 2026 14:11:08 +0200 Subject: [PATCH 8/8] fix: 3.5.x does not respect spark.checkpoint.dir --- python/tests/test_graphframes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tests/test_graphframes.py b/python/tests/test_graphframes.py index 6e8d1189c..d372cb903 100644 --- a/python/tests/test_graphframes.py +++ b/python/tests/test_graphframes.py @@ -354,7 +354,7 @@ def test_bfs(local_g: GraphFrame) -> None: def test_all_paths(local_g: GraphFrame) -> None: # local_g: A->B (love), B->A (hate), B->C (follow) # Directed: A can reach C via A->B->C (1 path) - paths = local_g.all_paths("name='A'", "name='C'") + paths = local_g.all_paths("name='A'", "name='C'", use_local_checkpoints=True) assert paths is not None assert {"path", "len"}.issubset(set(paths.columns)) paths_list = paths.collect() @@ -362,16 +362,16 @@ def test_all_paths(local_g: GraphFrame) -> None: assert paths_list[0]["len"] == 2 # With edge filter that removes the 'follow' edge: no path A->C - paths_filtered = local_g.all_paths("name='A'", "name='C'", edge_filter="action!='follow'") + paths_filtered = local_g.all_paths("name='A'", "name='C'", edge_filter="action!='follow'", use_local_checkpoints=True) assert paths_filtered.count() == 0 # Undirected: A->B->C and A->B->A (no simple path to C via A again), # but also C->B->A is now possible, still only A->B->C from A to C - paths_undirected = local_g.all_paths("name='A'", "name='C'", is_directed=False) + paths_undirected = local_g.all_paths("name='A'", "name='C'", is_directed=False, use_local_checkpoints=True) assert paths_undirected.count() >= 1 # max_path_length too short: no paths - paths_short = local_g.all_paths("name='A'", "name='C'", max_path_length=1) + paths_short = local_g.all_paths("name='A'", "name='C'", max_path_length=1, use_local_checkpoints=True) assert paths_short.count() == 0