From 3ace902d5670b02fbeb36f30b09dd56ab7aeaf5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:17:42 +0000 Subject: [PATCH 01/19] GraphFrame.scala:711 --- src/main/scala/org/graphframes/GraphFrame.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 523026092..956d0a4ca 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -708,7 +708,7 @@ object GraphFrame extends Serializable with Logging { def fromEdges(e: DataFrame): GraphFrame = { val srcs = e.select(e("src").as("id")) val dsts = e.select(e("dst").as("id")) - val v = srcs.unionAll(dsts).distinct + val v = srcs.unionAll(dsts).distinct() v.persist(StorageLevel.MEMORY_AND_DISK) apply(v, e) } From d57b55c8df0a6e4487c226b61b3c975502dffdeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:28:06 +0000 Subject: [PATCH 02/19] BeliefPropagation.scala --- .../scala/org/graphframes/examples/BeliefPropagation.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/examples/BeliefPropagation.scala b/src/main/scala/org/graphframes/examples/BeliefPropagation.scala index 4c75d8369..9074bd13c 100644 --- a/src/main/scala/org/graphframes/examples/BeliefPropagation.scala +++ b/src/main/scala/org/graphframes/examples/BeliefPropagation.scala @@ -129,7 +129,7 @@ object BeliefPropagation { def runBPwithGraphX(g: GraphFrame, numIter: Int): GraphFrame = { // Choose colors for vertices for BP scheduling. val colorG = colorGraph(g) - val numColors: Int = colorG.vertices.select("color").distinct.count().toInt + val numColors: Int = colorG.vertices.select("color").distinct().count().toInt // Convert GraphFrame to GraphX, and initialize beliefs. val gx0 = colorG.toGraphX @@ -206,7 +206,7 @@ object BeliefPropagation { def runBPwithGraphFrames(g: GraphFrame, numIter: Int): GraphFrame = { // Choose colors for vertices for BP scheduling. val colorG = colorGraph(g) - val numColors: Int = colorG.vertices.select("color").distinct.count().toInt + val numColors: Int = colorG.vertices.select("color").distinct().count().toInt // TODO: Handle vertices without any edges. From feaaa4623f24eb4ef0edae3e930b41418516d7ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:35:40 +0000 Subject: [PATCH 03/19] GraphFrame.scala:805 --- src/main/scala/org/graphframes/GraphFrame.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 956d0a4ca..5e2d0d645 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -802,7 +802,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - s.fieldNames.map(f => col + "." + f) + scala.collection.immutable.ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f) case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 6b14b9528430f1b9912f2d0e4061770b608b6aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:42:52 +0000 Subject: [PATCH 04/19] ArraySeq.unsafeWrapArray --- src/main/scala/org/graphframes/GraphFrame.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 5e2d0d645..361e48334 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -20,6 +20,7 @@ package org.graphframes import java.util.Random import scala.reflect.runtime.universe.TypeTag +import scala.collection.immutable.ArraySeq import org.apache.spark.graphx.{Edge, Graph} import org.apache.spark.sql._ @@ -802,7 +803,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - scala.collection.immutable.ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f) + ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f) case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 1006458f20a0000e4d684250f7e74c8cd543f259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:46:15 +0000 Subject: [PATCH 05/19] mutable.WrappedArray --- src/main/scala/org/graphframes/GraphFrame.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 361e48334..721ce2282 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -20,7 +20,7 @@ package org.graphframes import java.util.Random import scala.reflect.runtime.universe.TypeTag -import scala.collection.immutable.ArraySeq +import scala.collection.mutable.WrappedArray import org.apache.spark.graphx.{Edge, Graph} import org.apache.spark.sql._ @@ -803,7 +803,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f) + WrappedArray.make(s.fieldNames).map(f => col + "." + f) case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 66a6b8adb7904fbfa1ad33888f62c17fada4ea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:49:52 +0000 Subject: [PATCH 06/19] .toSeq --- src/main/scala/org/graphframes/GraphFrame.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 721ce2282..3ac8d8f17 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -803,7 +803,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - WrappedArray.make(s.fieldNames).map(f => col + "." + f) + WrappedArray.make(s.fieldNames).map(f => col + "." + f).toSeq case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 0ba61776517768e44e17d2ac89df3c806f5c9bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 19:59:38 +0000 Subject: [PATCH 07/19] ArraySeq --- src/main/scala/org/graphframes/GraphFrame.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 3ac8d8f17..b1d84809e 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -803,7 +803,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - WrappedArray.make(s.fieldNames).map(f => col + "." + f).toSeq + scala.collection.immutable.ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f).toSeq case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 24dbf51104ceaa17b3bebc526bea31bfc87f046a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 20:10:48 +0000 Subject: [PATCH 08/19] toIndexedSeq --- src/main/scala/org/graphframes/GraphFrame.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index b1d84809e..d28a4496e 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -3,8 +3,8 @@ * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -803,7 +803,7 @@ object GraphFrame extends Serializable with Logging { private[graphframes] def colStar(df: DataFrame, col: String): Seq[String] = { df.schema(col).dataType match { case s: StructType => - scala.collection.immutable.ArraySeq.unsafeWrapArray(s.fieldNames).map(f => col + "." + f).toSeq + s.fieldNames.map(f => col + "." + f).toIndexedSeq case other => throw new RuntimeException(s"Unknown error in GraphFrame. Expected column $col to be" + s" StructType, but found type: $other") From 547ec046f4ceecdefa112277bceda346c54753c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 20:24:21 +0000 Subject: [PATCH 09/19] graphs .toIndexedSeq --- src/main/scala/org/graphframes/examples/Graphs.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/examples/Graphs.scala b/src/main/scala/org/graphframes/examples/Graphs.scala index c929c61ec..224ab1224 100644 --- a/src/main/scala/org/graphframes/examples/Graphs.scala +++ b/src/main/scala/org/graphframes/examples/Graphs.scala @@ -150,7 +150,7 @@ class Graphs private[graphframes] () { |4,2,5.0 |4,3,1.0 |4,4,5.0 - """.stripMargin.split("\n").map(_.trim).filterNot(_.isEmpty) + """.stripMargin.split("\n").map(_.trim).filterNot(_.isEmpty).toIndexedSeq /** * This method generates a grid Ising model with random parameters. From a4d295a11853dd7df63ee2a6a9bc70b4c95cecbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 20:36:00 +0000 Subject: [PATCH 10/19] als_data toIndexedSeq --- src/main/scala/org/graphframes/examples/Graphs.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/examples/Graphs.scala b/src/main/scala/org/graphframes/examples/Graphs.scala index 224ab1224..d6672b69f 100644 --- a/src/main/scala/org/graphframes/examples/Graphs.scala +++ b/src/main/scala/org/graphframes/examples/Graphs.scala @@ -122,7 +122,7 @@ class Graphs private[graphframes] () { */ def ALSSyntheticData(): GraphFrame = { val sc = spark.sparkContext - val data = sc.parallelize(als_data).map { line => + val data = sc.parallelize(als_data.toIndexedSeq).map { line => val fields = line.split(",") (fields(0).toLong * 2, fields(1).toLong * 2 + 1, fields(2).toDouble) } @@ -150,7 +150,7 @@ class Graphs private[graphframes] () { |4,2,5.0 |4,3,1.0 |4,4,5.0 - """.stripMargin.split("\n").map(_.trim).filterNot(_.isEmpty).toIndexedSeq + """.stripMargin.split("\n").map(_.trim).filterNot(_.isEmpty) /** * This method generates a grid Ising model with random parameters. From b9c917f1b83be9df05afb3a6db0fb352640582ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 20:49:11 +0000 Subject: [PATCH 11/19] val vs toIndexedSeq --- src/main/scala/org/graphframes/examples/Graphs.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/examples/Graphs.scala b/src/main/scala/org/graphframes/examples/Graphs.scala index d6672b69f..2ea19e525 100644 --- a/src/main/scala/org/graphframes/examples/Graphs.scala +++ b/src/main/scala/org/graphframes/examples/Graphs.scala @@ -127,7 +127,7 @@ class Graphs private[graphframes] () { (fields(0).toLong * 2, fields(1).toLong * 2 + 1, fields(2).toDouble) } val edges = spark.createDataFrame(data).toDF("src", "dst", "weight") - val vs = data.flatMap(r => r._1 :: r._2 :: Nil).collect().distinct.map(x => Tuple1(x)) + val vs = data.flatMap(r => r._1 :: r._2 :: Nil).collect().distinct.map(x => Tuple1(x)).toIndexedSeq val vertices = spark.createDataFrame(vs).toDF("id") GraphFrame(vertices, edges) } From 820e4140b8ff30a88cbc2d68eb087b3b2fb2ca3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 20:55:49 +0000 Subject: [PATCH 12/19] scala.jdk.CollectionConverters --- src/main/scala/org/graphframes/lib/ShortestPaths.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/graphframes/lib/ShortestPaths.scala b/src/main/scala/org/graphframes/lib/ShortestPaths.scala index f110d6f1f..75d48810d 100644 --- a/src/main/scala/org/graphframes/lib/ShortestPaths.scala +++ b/src/main/scala/org/graphframes/lib/ShortestPaths.scala @@ -19,7 +19,7 @@ package org.graphframes.lib import java.util -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.apache.spark.graphx.{lib => graphxlib} import org.apache.spark.sql.{Column, DataFrame, Row} From 1c13bafc84f39657af8f5556e030617ed08640dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:07:57 +0000 Subject: [PATCH 13/19] graphFrame toIndexedSeq --- src/main/scala/org/graphframes/GraphFrame.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index d28a4496e..070dd44d1 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -53,8 +53,8 @@ class GraphFrame private( override def toString: String = { // We call select on the vertices and edges to ensure that ID, SRC, DST always come first // in the printed schema. - val v = vertices.select(ID, vertices.columns.filter(_ != ID) :_ *).toString - val e = edges.select(SRC, DST +: edges.columns.filter(c => c != SRC && c != DST) :_ *).toString + val v = vertices.select(ID +: vertices.columns.filter(_ != ID).toIndexedSeq: _*).toString + val e = edges.select((SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq): _*).toString "GraphFrame(v:" + v + ", e:" + e + ")" } From 59b4ba1d96f17c9b49c2553b4c1d7e262f4bdfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:11:43 +0000 Subject: [PATCH 14/19] Split the column list --- src/main/scala/org/graphframes/GraphFrame.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 070dd44d1..d2c76a8e6 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -53,8 +53,10 @@ class GraphFrame private( override def toString: String = { // We call select on the vertices and edges to ensure that ID, SRC, DST always come first // in the printed schema. - val v = vertices.select(ID +: vertices.columns.filter(_ != ID).toIndexedSeq: _*).toString - val e = edges.select((SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq): _*).toString + val vCols = ID +: vertices.columns.filter(_ != ID).toIndexedSeq + val eCols = SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq + val v = vertices.select(vCols: _*).toString + val e = edges.select(eCols: _*).toString "GraphFrame(v:" + v + ", e:" + e + ")" } From 108951166d83e123c8b4976160b15a2a11c10a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:15:22 +0000 Subject: [PATCH 15/19] DataFrame.select --- src/main/scala/org/graphframes/GraphFrame.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index d2c76a8e6..f02bdebc9 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -53,10 +53,10 @@ class GraphFrame private( override def toString: String = { // We call select on the vertices and edges to ensure that ID, SRC, DST always come first // in the printed schema. - val vCols = ID +: vertices.columns.filter(_ != ID).toIndexedSeq - val eCols = SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq - val v = vertices.select(vCols: _*).toString - val e = edges.select(eCols: _*).toString + val vCols = (ID +: vertices.columns.filter(_ != ID).toIndexedSeq).map(col) + val eCols = (SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq).map(col) + val v = vertices.select(vCols.head, vCols.tail: _*).toString + val e = edges.select(eCols.head, eCols.tail: _*).toString "GraphFrame(v:" + v + ", e:" + e + ")" } From 4edbaeb81d39a05c389cc328adbdea6dfb36ab5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:19:55 +0000 Subject: [PATCH 16/19] Cols.toSeq --- src/main/scala/org/graphframes/GraphFrame.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index f02bdebc9..42c67606a 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -55,8 +55,8 @@ class GraphFrame private( // in the printed schema. val vCols = (ID +: vertices.columns.filter(_ != ID).toIndexedSeq).map(col) val eCols = (SRC +: DST +: edges.columns.filter(c => c != SRC && c != DST).toIndexedSeq).map(col) - val v = vertices.select(vCols.head, vCols.tail: _*).toString - val e = edges.select(eCols.head, eCols.tail: _*).toString + val v = vertices.select(vCols.toSeq: _*).toString + val e = edges.select(eCols.toSeq: _*).toString "GraphFrame(v:" + v + ", e:" + e + ")" } From c1680644011c14ffe43db79ed8d353de30fc6585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:41:59 +0000 Subject: [PATCH 17/19] Added .toSeq before : _* --- src/main/scala/org/graphframes/GraphFrame.scala | 2 +- src/main/scala/org/graphframes/lib/BFS.scala | 2 +- src/main/scala/org/graphframes/lib/GraphXConversions.scala | 6 +++--- src/main/scala/org/graphframes/lib/ShortestPaths.scala | 2 +- src/main/scala/org/graphframes/lib/TriangleCount.scala | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 42c67606a..89e79fcd0 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -814,7 +814,7 @@ object GraphFrame extends Serializable with Logging { /** Nest all columns within a single StructType column with the given name */ private[graphframes] def nestAsCol(df: DataFrame, name: String): Column = { - struct(df.columns.map(c => df(c)) :_*).as(name) + struct(df.columns.map(c => df(c)).toSeq: _*).as(name) } // ========== Motif finding ========== diff --git a/src/main/scala/org/graphframes/lib/BFS.scala b/src/main/scala/org/graphframes/lib/BFS.scala index df859c14d..80ebe313e 100644 --- a/src/main/scala/org/graphframes/lib/BFS.scala +++ b/src/main/scala/org/graphframes/lib/BFS.scala @@ -218,7 +218,7 @@ private object BFS extends Logging with Serializable { } } val ordered = paths.columns.sortBy(rank _) - paths.select(ordered.map(col): _*) + paths.select(ordered.map(col).toSeq: _*) } else { logInfo(s"GraphFrame.bfs failed to find a path of length <= $maxPathLength.") // Return empty DataFrame diff --git a/src/main/scala/org/graphframes/lib/GraphXConversions.scala b/src/main/scala/org/graphframes/lib/GraphXConversions.scala index 88e690876..13f9beb65 100644 --- a/src/main/scala/org/graphframes/lib/GraphXConversions.scala +++ b/src/main/scala/org/graphframes/lib/GraphXConversions.scala @@ -104,10 +104,10 @@ private[graphframes] object GraphXConversions { val otherFields = df.schema.fieldNames.filter(_ != structName).map(col) if (renamedSubfields.isEmpty) { // Do not attempt to add an empty structure. - df.select(otherFields : _*) + df.select(otherFields.toSeq: _*) } else { - val renamedStruct = struct(renamedSubfields : _*).as(structName) - df.select(renamedStruct +: otherFields : _*) + val renamedStruct = struct(renamedSubfields.toSeq: _*).as(structName) + df.select((renamedStruct +: otherFields).toSeq: _*) } } diff --git a/src/main/scala/org/graphframes/lib/ShortestPaths.scala b/src/main/scala/org/graphframes/lib/ShortestPaths.scala index 75d48810d..6b1f15c74 100644 --- a/src/main/scala/org/graphframes/lib/ShortestPaths.scala +++ b/src/main/scala/org/graphframes/lib/ShortestPaths.scala @@ -91,7 +91,7 @@ private object ShortestPaths { mapToLandmark(col(DISTANCE_ID)) } val cols = graph.vertices.columns.map(col) :+ distanceCol.as(DISTANCE_ID) - g.vertices.select(cols: _*) + g.vertices.select(cols.toSeq: _*) } private val DISTANCE_ID = "distances" diff --git a/src/main/scala/org/graphframes/lib/TriangleCount.scala b/src/main/scala/org/graphframes/lib/TriangleCount.scala index 816cb0da7..fb8c51462 100644 --- a/src/main/scala/org/graphframes/lib/TriangleCount.scala +++ b/src/main/scala/org/graphframes/lib/TriangleCount.scala @@ -68,7 +68,7 @@ private object TriangleCount { val v = graph.vertices val countsCol = when(col("count").isNull, 0L).otherwise(col("count")) val newV = v.join(triangleCounts, v(ID) === triangleCounts(ID), "left_outer") - .select(countsCol.as(COUNT_ID) +: v.columns.map(v.apply) :_ *) + .select((countsCol.as(COUNT_ID) +: v.columns.map(v.apply)).toSeq: _*) newV } From d3d63407bc38dc0fd85335dbd528f5a5a4f04e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:55:09 +0000 Subject: [PATCH 18/19] fix the --- src/main/scala/org/graphframes/GraphFrame.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 89e79fcd0..7c199f885 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -3,8 +3,8 @@ * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * From 003f6ec012198605b9403260148fa967801ab6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 15 Jan 2025 21:58:36 +0000 Subject: [PATCH 19/19] remove WrappedArray in graphframe --- src/main/scala/org/graphframes/GraphFrame.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/src/main/scala/org/graphframes/GraphFrame.scala index 7c199f885..c9a221a6a 100644 --- a/src/main/scala/org/graphframes/GraphFrame.scala +++ b/src/main/scala/org/graphframes/GraphFrame.scala @@ -20,7 +20,6 @@ package org.graphframes import java.util.Random import scala.reflect.runtime.universe.TypeTag -import scala.collection.mutable.WrappedArray import org.apache.spark.graphx.{Edge, Graph} import org.apache.spark.sql._