From 8b8b1d3a72cd1342adae5ce823dbc29b4b6a57c2 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Tue, 16 Jun 2026 10:58:21 -0700 Subject: [PATCH 1/3] ci: add codespell spell check and extend python linting to dev scripts - Add codespell (config in .codespellrc) covering yml, python and scala via a new Spell Check workflow and a pre-commit hook; fix the typos it surfaced across sources and docs. graphx/ is excluded to stay aligned with upstream Apache Spark. - Extend black/flake8/isort to the dev/ and python/dev/ scripts in both the Python CI code-style step and pre-commit. --- .codespellrc | 12 ++++++++++++ .github/workflows/python-ci.yml | 6 +++--- .github/workflows/spellcheck.yml | 17 +++++++++++++++++ .pre-commit-config.yaml | 12 +++++++++--- CONTRIBUTING.md | 3 ++- README.md | 2 +- .../main/scala/org/graphframes/GraphFrame.scala | 4 ++-- .../embeddings/RandomWalkEmbeddings.scala | 4 ++-- .../scala/org/graphframes/lib/AllPaths.scala | 2 +- .../scala/org/graphframes/lib/HyperANF.scala | 2 +- .../org/graphframes/lib/LabelPropagation.scala | 2 +- .../graphframes/lib/MaximalIndependentSet.scala | 2 +- .../graphframes/lib/RandomizedContraction.scala | 2 +- .../org/graphframes/lib/ShortestPaths.scala | 2 +- .../src/main/scala/org/graphframes/mixins.scala | 2 +- .../org/graphframes/pattern/patterns.scala | 2 +- .../org/graphframes/rw/RandomWalkBase.scala | 4 ++-- .../graphframes/embeddings/Hash2VecSuite.scala | 2 +- .../lib/AggregateMessagesSuite.scala | 2 +- .../org/graphframes/pattern/PatternSuite.scala | 4 ++-- docs/mdoc/01-installation.md | 4 ++-- docs/src/02-quick-start/02-quick-start.md | 2 +- docs/src/03-tutorials/02-motif-tutorial.md | 6 +++--- docs/src/04-user-guide/03-centralities.md | 4 ++-- docs/src/04-user-guide/04-motif-finding.md | 4 ++-- docs/src/04-user-guide/05-traversals.md | 8 ++++---- docs/src/04-user-guide/06-graph-clustering.md | 8 ++++---- docs/src/05-blog/996-graphframes-012-release.md | 2 +- docs/src/05-blog/998-graphframes-010-release.md | 4 ++-- .../06-contributing/01-contributing-guide.md | 4 ++-- docs/src/llms.txt | 2 +- python/dev/build_jar.py | 4 +--- python/docs/conf.py | 2 +- python/graphframes/examples/graphs.py | 2 +- python/graphframes/graphframe.py | 4 ++-- 35 files changed, 91 insertions(+), 57 deletions(-) create mode 100644 .codespellrc create mode 100644 .github/workflows/spellcheck.yml diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 000000000..800d07b84 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,12 @@ +[codespell] +# Skip generated, vendored, binary, and lock files. +# graphx/ is ported from Apache Spark GraphX; keep it aligned with upstream. +# graphframes-connect-databricks/ holds local sbt build output (gitignored). +skip = .git,*.lock,poetry.lock,*.iml,target,build,dist,*_pb2*.py,*_pb2_grpc.py,graphx,graphframes-connect-databricks,*.svg,*.png,*.gif,*.parquet,*.csv,*.json,*.pom,*.semanticdb +# Identifiers and domain terms that are not misspellings: +# mis/MIS - Maximal Independent Set +# te/ser - local variable names in Scala tests/algorithms +# fof/mye/protocall/alledges - local variable names (camelCase) in tests +# afterall - ScalaTest `afterAll` lifecycle override +# jurney - maintainer surname (Russell Jurney) +ignore-words-list = mis,te,ser,fof,mye,protocall,alledges,afterall,jurney diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 40ad81a5d..0f88f6028 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -57,9 +57,9 @@ jobs: - name: Code style working-directory: ./python run: | - poetry run python -m black --check graphframes - poetry run python -m flake8 graphframes - poetry run python -m isort --check graphframes + poetry run python -m black --check graphframes dev ../dev + poetry run python -m flake8 graphframes dev ../dev + poetry run python -m isort --check graphframes dev ../dev - name: Build jar working-directory: ./python run: | diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 000000000..c6f76d17d --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,17 @@ +name: Spell Check +on: [push, pull_request] +permissions: + contents: read +jobs: + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install codespell + run: pip install codespell==2.4.2 + - name: Run codespell + # Configuration (skips and ignore list) lives in .codespellrc. + run: codespell . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a1c2099e..a6163616d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,19 +3,19 @@ repos: hooks: - id: black name: black - entry: bash -c 'cd python && poetry run black graphframes/' -- + entry: bash -c 'cd python && poetry run black graphframes/ dev/ ../dev/' -- language: system types: [python] - id: flake8 name: flake8 - entry: bash -c 'cd python && poetry run flake8 graphframes/' -- + entry: bash -c 'cd python && poetry run flake8 graphframes/ dev/ ../dev/' -- language: system types: [python] - id: isort name: isort - entry: bash -c 'cd python && poetry run isort graphframes/' -- + entry: bash -c 'cd python && poetry run isort graphframes/ dev/ ../dev/' -- language: system types: [python] @@ -32,3 +32,9 @@ repos: language: system types: [scala] pass_filenames: false + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.2 + hooks: + - id: codespell + # Configuration (skips and ignore list) lives in .codespellrc. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c51fc4fa..96bdab928 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,7 +117,8 @@ Enhancement suggestions are tracked as [GitHub issues](https://github.com/graphf We are using the following tools for enforcing of the codestyle: - for python we are relying on the `black` + `flake8` + `isort`; -- for scala we are relying on the `scalafmt`. +- for scala we are relying on the `scalafmt`; +- for spelling (across yml, python, and scala) we are relying on `codespell` (configured in `.codespellrc`). You can enforce the code style using the [pre-commit-hooks](https://pre-commit.com/): diff --git a/README.md b/README.md index 29483a58e..829cf79fd 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ There are some popular use cases when GraphFrames is almost irreplaceable, inclu - Compliance analytics with a scalable shortest paths algorithm and motif analysis; - Anti-fraud with scalable cycles detection in large networks and by using K-Core algorithm; - Identity resolution at the scale of billions with highly efficient connected components; -- Plan marketing campaigns in social networks using Maximal Indpendent Set algorithm; +- Plan marketing campaigns in social networks using Maximal Independent Set algorithm; - Rank search result with a distributed, Pregel-based PageRank; - Cluster huge graphs with Label Propagation and Power Iteration Clustering; - Compute node embeddings at billion scale using Random-Walks and Hash2Vec model; diff --git a/core/src/main/scala/org/graphframes/GraphFrame.scala b/core/src/main/scala/org/graphframes/GraphFrame.scala index d5c59e534..9b44b11a5 100644 --- a/core/src/main/scala/org/graphframes/GraphFrame.scala +++ b/core/src/main/scala/org/graphframes/GraphFrame.scala @@ -623,7 +623,7 @@ class GraphFrame private ( case VarLengthPattern(src, name, min, max, direction, dst) => if (min.isEmpty || max.isEmpty) { throw new InvalidParseException( - s"Unbounded length patten ${pattern} is not supported! " + + s"Unbounded length pattern ${pattern} is not supported! " + "Please a pattern of defined length.") } findVarLengthPattern(src, name, min.toInt, max.toInt, direction, dst) @@ -982,7 +982,7 @@ class GraphFrame private ( * large-scale sparse graphs." Proceedings of Simpósio Brasileiro de Pesquisa Operacional * (SBPO’15) (2015): 1-11. * - * Returns a DataFrame with unque cycles. + * Returns a DataFrame with unique cycles. * * @return * an instance of DetectingCycles initialized with the current context diff --git a/core/src/main/scala/org/graphframes/embeddings/RandomWalkEmbeddings.scala b/core/src/main/scala/org/graphframes/embeddings/RandomWalkEmbeddings.scala index fb140d993..2624fea66 100644 --- a/core/src/main/scala/org/graphframes/embeddings/RandomWalkEmbeddings.scala +++ b/core/src/main/scala/org/graphframes/embeddings/RandomWalkEmbeddings.scala @@ -230,7 +230,7 @@ class RandomWalkEmbeddings private[graphframes] (private val graph: GraphFrame) .run() } else { // we need to create a new DataFrame, so unpersisting peristedEmbeddings - // does not accidently unpersist the result; + // does not accidentally unpersist the result; // dummy operations are cheap, but will create a different plan. persistedEmbeddings .withColumnRenamed(RandomWalkEmbeddings.embeddingColName, "x") @@ -269,7 +269,7 @@ object RandomWalkEmbeddings extends Serializable { * provide a smooth way to initialize the whole embeddings pipeline with a single method call * that is usable for Python API (py4j and Spark Connect). * - * Instead of this API it is recommended to use new + settters of the class! + * Instead of this API it is recommended to use new + setters of the class! */ def pythonAPI( graph: GraphFrame, diff --git a/core/src/main/scala/org/graphframes/lib/AllPaths.scala b/core/src/main/scala/org/graphframes/lib/AllPaths.scala index 279b8d01a..c9ae0ee52 100644 --- a/core/src/main/scala/org/graphframes/lib/AllPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/AllPaths.scala @@ -49,7 +49,7 @@ import org.graphframes.WithLocalCheckpoints * - `len`: number of edges in the path (Long) * * Note: in the case of undirected graph an algorithm run on the internal graph made by union - * edges and reversed edges. It is assummed that graph does not have multi-edges. Results may be + * edges and reversed edges. It is assumed that graph does not have multi-edges. Results may be * unstable and unpredictable for the graph with multi-edges. */ class AllPaths private[graphframes] (private val graph: GraphFrame) diff --git a/core/src/main/scala/org/graphframes/lib/HyperANF.scala b/core/src/main/scala/org/graphframes/lib/HyperANF.scala index 2f752d5fa..57603b1db 100644 --- a/core/src/main/scala/org/graphframes/lib/HyperANF.scala +++ b/core/src/main/scala/org/graphframes/lib/HyperANF.scala @@ -171,7 +171,7 @@ class HyperANF private[graphframes] (graph: GraphFrame) .groupBy(col(GraphFrame.SRC).alias(GraphFrame.ID)) .agg(hll_union_agg(s"hop_${hop - 1}").alias(s"hop_${hop}")) - // stanard GF persist-unpersist-checkpoint flow + // standard GF persist-unpersist-checkpoint flow state = { val stateToPersist = state.join(nState, GraphFrame.ID) if (shouldCheckpoint && hop % checkpointInterval == 0) { diff --git a/core/src/main/scala/org/graphframes/lib/LabelPropagation.scala b/core/src/main/scala/org/graphframes/lib/LabelPropagation.scala index 6cc0866e4..2c0065953 100644 --- a/core/src/main/scala/org/graphframes/lib/LabelPropagation.scala +++ b/core/src/main/scala/org/graphframes/lib/LabelPropagation.scala @@ -103,7 +103,7 @@ private object LabelPropagation { // Overall: // - Initial labels - IDs // - Active vertex col (halt voting) - did the label changed? - // - Choosing a new label - top across neighbours (tie-braking is determenistic) + // - Choosing a new label - top across neighbours (tie-braking is deterministic) val preparedGraph = GraphFrame( graph.vertices.select(GraphFrame.ID), diff --git a/core/src/main/scala/org/graphframes/lib/MaximalIndependentSet.scala b/core/src/main/scala/org/graphframes/lib/MaximalIndependentSet.scala index 41cca2d0c..ed4882ee4 100644 --- a/core/src/main/scala/org/graphframes/lib/MaximalIndependentSet.scala +++ b/core/src/main/scala/org/graphframes/lib/MaximalIndependentSet.scala @@ -150,7 +150,7 @@ object MaximalIndependentSet extends Serializable with Logging { val joinedMIS = isJoinedMIS.filter((!col(notJoinedMISCol)) && col(isNominated)).select(GraphFrame.ID) - // update curent MIS + // update current MIS val updatedMIS = misDF .join( isolatedVertices.select(col(GraphFrame.ID), lit(true).alias("f")), diff --git a/core/src/main/scala/org/graphframes/lib/RandomizedContraction.scala b/core/src/main/scala/org/graphframes/lib/RandomizedContraction.scala index 90459ccf2..a9fec51a6 100644 --- a/core/src/main/scala/org/graphframes/lib/RandomizedContraction.scala +++ b/core/src/main/scala/org/graphframes/lib/RandomizedContraction.scala @@ -168,7 +168,7 @@ private[graphframes] object RandomizedContraction extends Logging with Serializa } logInfo(s"graph was successfully contracted for $iter iterations") - logInfo("start reverse tranformation") + logInfo("start reverse transformation") var accA = 1L var accB = 0L diff --git a/core/src/main/scala/org/graphframes/lib/ShortestPaths.scala b/core/src/main/scala/org/graphframes/lib/ShortestPaths.scala index ef3ff77d0..a3cb085ef 100644 --- a/core/src/main/scala/org/graphframes/lib/ShortestPaths.scala +++ b/core/src/main/scala/org/graphframes/lib/ShortestPaths.scala @@ -139,7 +139,7 @@ private object ShortestPaths extends Logging { // For landmark vertices the initial distance to itself is set to 0 // Example: graph with vertices a, b, c, d; landmarks = (c, d) - // we shoudl init the following: + // we should init the following: // (a, Map()), (b, Map()), (c, Map(c -> 0)), (d, Map(d -> 0)) // // Inside the following function it is done by applying multiple case-when diff --git a/core/src/main/scala/org/graphframes/mixins.scala b/core/src/main/scala/org/graphframes/mixins.scala index 59fb96b8b..907018052 100644 --- a/core/src/main/scala/org/graphframes/mixins.scala +++ b/core/src/main/scala/org/graphframes/mixins.scala @@ -222,7 +222,7 @@ private[graphframes] trait WithDirection { } /** - * Gets should graph be considred as directed. + * Gets should graph be considered as directed. * * @return * true if directed diff --git a/core/src/main/scala/org/graphframes/pattern/patterns.scala b/core/src/main/scala/org/graphframes/pattern/patterns.scala index 357f686b4..6571c3a88 100644 --- a/core/src/main/scala/org/graphframes/pattern/patterns.scala +++ b/core/src/main/scala/org/graphframes/pattern/patterns.scala @@ -67,7 +67,7 @@ private[graphframes] object Pattern { } /** - * Rewirte a motif string if there are incomming edges + * Rewrite a motif string if there are incoming edges */ private[graphframes] def rewriteIncomingEdges(patterns: String): String = { val reversedEdge = diff --git a/core/src/main/scala/org/graphframes/rw/RandomWalkBase.scala b/core/src/main/scala/org/graphframes/rw/RandomWalkBase.scala index 2638df74c..d30d31200 100644 --- a/core/src/main/scala/org/graphframes/rw/RandomWalkBase.scala +++ b/core/src/main/scala/org/graphframes/rw/RandomWalkBase.scala @@ -54,7 +54,7 @@ trait RandomWalkBase extends Serializable with Logging with WithIntermediateStor /** Unique identifier for the current random walk run. */ protected var runID: String = java.util.UUID.randomUUID().toString - /** Starting batch index for continous mode */ + /** Starting batch index for continuous mode */ protected var startingIteration: Int = 1 /** Internal handler of vertex data type */ @@ -187,7 +187,7 @@ trait RandomWalkBase extends Serializable with Logging with WithIntermediateStor def getRunId(): String = runID /** - * Sets the startng batch index for the continous mode. See @setWalkId comment for details. + * Sets the startng batch index for the continuous mode. See @setWalkId comment for details. * * @param value * @return diff --git a/core/src/test/scala/org/graphframes/embeddings/Hash2VecSuite.scala b/core/src/test/scala/org/graphframes/embeddings/Hash2VecSuite.scala index 859227bcc..423cdb464 100644 --- a/core/src/test/scala/org/graphframes/embeddings/Hash2VecSuite.scala +++ b/core/src/test/scala/org/graphframes/embeddings/Hash2VecSuite.scala @@ -56,7 +56,7 @@ class Hash2VecSuite extends SparkFunSuite with GraphFrameTestSparkContext with B assert(collected.length === uniqueElementsCnt) } - test("hash2vec reproducable with seed") { + test("hash2vec reproducible with seed") { val hash2vecResults = new Hash2Vec().setSequenceCol("seq").setHashingSeed(42).run(longSequences) val hash2vecResults2 = diff --git a/core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala b/core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala index a026f1bbd..53687e5aa 100644 --- a/core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala +++ b/core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala @@ -81,7 +81,7 @@ class AggregateMessagesSuite extends SparkFunSuite with GraphFrameTestSparkConte aggMap.keys.foreach { case user => assert(aggMap(user) === trueAgg(user), s"Failure on user $user") } - // Perfom the aggregation again, this time providing the messages as Strings instead. + // Perform the aggregation again, this time providing the messages as Strings instead. val msgToSrc2 = "(dst['age'] + CASE WHEN (edge['relationship'] = 'friend') THEN 1 ELSE 0 END)" val msgToDst2 = "src['age']" val agg2 = g.aggregateMessages diff --git a/core/src/test/scala/org/graphframes/pattern/PatternSuite.scala b/core/src/test/scala/org/graphframes/pattern/PatternSuite.scala index d97309c4e..a4c503d7f 100644 --- a/core/src/test/scala/org/graphframes/pattern/PatternSuite.scala +++ b/core/src/test/scala/org/graphframes/pattern/PatternSuite.scala @@ -116,7 +116,7 @@ class PatternSuite extends SparkFunSuite { UndirectedEdge(AnonymousEdge(NamedVertex("v"), NamedVertex("k"))))) } - test("rewrite incomming edges") { + test("rewrite incoming edges") { assert(Pattern.rewriteIncomingEdges("(u)<-[e]-(v);") === "(v)-[e]->(u)") assert(Pattern.rewriteIncomingEdges("!(u)<-[e]-(v);") === "!(v)-[e]->(u)") assert( @@ -130,7 +130,7 @@ class PatternSuite extends SparkFunSuite { "(v1)<-[e*1..2]->(v2)") === "(v1)-[e*1..2]->(v2);(v2)-[e*1..2]->(v1)") } - test("rewrite incomming edges and parse") { + test("rewrite incoming edges and parse") { Pattern.parse("(v)<-[e]-(u)") === Pattern.parse("(u)-[e]->(v)") Pattern.parse("(v)<-[]-(u)") === Pattern.parse("(u)-[]->(v)") Pattern.parse("!(v)<-[]-(u)") === Pattern.parse("!(u)-[]->(v)") diff --git a/docs/mdoc/01-installation.md b/docs/mdoc/01-installation.md index ce6935cc8..3488d6797 100644 --- a/docs/mdoc/01-installation.md +++ b/docs/mdoc/01-installation.md @@ -4,7 +4,7 @@ If you are new to using Apache Spark, refer to the [Apache Spark Documentation]( ## Maven Central Coordinates -GraphFrames core is [published](https://central.sonatype.com/namespace/io.graphframes) in the Maven Central under namespace `io.graphframes`. All the artifacts are groupped using the following logic. +GraphFrames core is [published](https://central.sonatype.com/namespace/io.graphframes) in the Maven Central under namespace `io.graphframes`. All the artifacts are grouped using the following logic. ``` graphframes-{component-name}-{spark-major-version}_{scala-version} @@ -144,7 +144,7 @@ The only runtime dependency of the GraphFrames core is `graphframes-graphx-*`. I ### Microsoft Fabric Note -It was [reported](https://github.com/graphframes/graphframes/issues/748) that Microsft Fabric cannot resolve runtime dependencies of JVM pckages automatically. MS Fabric users should specify both core and `graphframes-graphx-*` (depends on the Apache Spark version). +It was [reported](https://github.com/graphframes/graphframes/issues/748) that Microsoft Fabric cannot resolve runtime dependencies of JVM packages automatically. MS Fabric users should specify both core and `graphframes-graphx-*` (depends on the Apache Spark version). ## Building GraphFrames from Source diff --git a/docs/src/02-quick-start/02-quick-start.md b/docs/src/02-quick-start/02-quick-start.md index 08a3d47e4..bc44490b1 100644 --- a/docs/src/02-quick-start/02-quick-start.md +++ b/docs/src/02-quick-start/02-quick-start.md @@ -97,7 +97,7 @@ supported algorithms: | Triangle Count | Yes | Yes | GF provides smoother API | | SVD++ | Yes | No | GX | | Cycles Detection | No | Yes | GF | -| Triangel Count | No | Yes | GF | +| Triangle Count | No | Yes | GF | | K-Core | No | Yes | GF | | Maximal Independent Set | No | Yes | GF | | Approximate Neighbor Functions | No | Yes | GF | diff --git a/docs/src/03-tutorials/02-motif-tutorial.md b/docs/src/03-tutorials/02-motif-tutorial.md index 16aaaa567..047f3f438 100644 --- a/docs/src/03-tutorials/02-motif-tutorial.md +++ b/docs/src/03-tutorials/02-motif-tutorial.md @@ -295,7 +295,7 @@ Edge count: 97,104 == Valid edge count: 97,104 Let's look for a simple motif: a directed triangle. We will find all instances of a directed triangle in the graph. The @pydoc(graphframes.GraphFrame.find) method takes a string as an argument that specifies the structure of a motif one edge at a time, in the same syntax as Cypher, with a semi-colon between edges. For a triangle motif, that works out to: `(a)-[e]->(b); (b)-[e2]->(c); (c)-[e3]->(a)`. Edge labels are optional, this is valid graph query: `(a)-[]->(b)`. -The `g.find()` method returns a `DataFrame` with fields fo each of the node and edge labels in the pattern. To further express the motif you're interested in, you can now use relational `DataFrame` operations to filter, group, and aggregate the results. This makes the network motif finding in GraphFrames very powerful, and this type of property graph motif was originally defined in the [graphframes paper](https://people.eecs.berkeley.edu/~matei/papers/2016/grades_graphframes.pdf). +The `g.find()` method returns a `DataFrame` with fields for each of the node and edge labels in the pattern. To further express the motif you're interested in, you can now use relational `DataFrame` operations to filter, group, and aggregate the results. This makes the network motif finding in GraphFrames very powerful, and this type of property graph motif was originally defined in the [graphframes paper](https://people.eecs.berkeley.edu/~matei/papers/2016/grades_graphframes.pdf). A complete description of the graph query language is in the [GraphFrames User Guide](/04-user-guide/04-motif-finding.md). Let's look at an example: a directed triangle. We will find all instances of a directed triangle in the graph. @@ -428,9 +428,9 @@ The result is a count of the divergent triangles in the graph by type. ## Property Graph Motifs -Simple motif finding can be used to explore a knowledge graph. It is also possibel to use domain knowledge to define and match known patterns and then explore new variant motifs. This can be used to apply and then expand domain knowledge about a knowledge graph. It is powerful stuff! +Simple motif finding can be used to explore a knowledge graph. It is also possible to use domain knowledge to define and match known patterns and then explore new variant motifs. This can be used to apply and then expand domain knowledge about a knowledge graph. It is powerful stuff! -We can do more with the properties of paths than just count them by node and edge type. We can use the properties of the nodes and edges in the paths to filter, group, and aggregate the results to form *property graph motifs*. Such complex motifs were first defined (without being formally named) in the paper describing this prject: [GraphFrames: An Integrated API for Mixing Graph and Relational Queries, Dave et al. 2016](https://people.eecs.berkeley.edu/~matei/papers/2016/grades_graphframes.pdf). They are a combination of graph and relational queries. We can use them to find complex patterns in the graph. +We can do more with the properties of paths than just count them by node and edge type. We can use the properties of the nodes and edges in the paths to filter, group, and aggregate the results to form *property graph motifs*. Such complex motifs were first defined (without being formally named) in the paper describing this project: [GraphFrames: An Integrated API for Mixing Graph and Relational Queries, Dave et al. 2016](https://people.eecs.berkeley.edu/~matei/papers/2016/grades_graphframes.pdf). They are a combination of graph and relational queries. We can use them to find complex patterns in the graph. The larger motifs get, the more interesting they are. Five nodes is often the limit with a Spark cluster, depending on how large your graph is. In this instance I will limit myself to a 4-path pattern as you may not have a Spark cluster on which to learn. Keep in mind that I am talking about paths - through aggregation a motif might cover thousands of nodes! diff --git a/docs/src/04-user-guide/03-centralities.md b/docs/src/04-user-guide/03-centralities.md index a2843853a..756b044c5 100644 --- a/docs/src/04-user-guide/03-centralities.md +++ b/docs/src/04-user-guide/03-centralities.md @@ -137,11 +137,11 @@ For `graphframes` only. To avoid exponential growing of the Spark' Logical Plan, - `use_local_checkpoints` -For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are realiable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. +For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are reliable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. - `storage_level` -The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarious, using `DISK_ONLY` may be faster. +The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarios, using `DISK_ONLY` may be faster. ### Python API diff --git a/docs/src/04-user-guide/04-motif-finding.md b/docs/src/04-user-guide/04-motif-finding.md index aeaf330a5..2339618f3 100644 --- a/docs/src/04-user-guide/04-motif-finding.md +++ b/docs/src/04-user-guide/04-motif-finding.md @@ -12,7 +12,7 @@ DSL for expressing structural patterns: square brackets `[e]`. * A pattern is expressed as a join of edges. Edge patterns can be joined with semicolons. Motif `"(a)-[e1]->(b); (b)-[e2]->(c)"` specifies two edges from `a` to `b` to `c`. -* Simply, you can also quantify the fixed length like `"(a)-[e*2]->(c)"`. The motif parser decompose it into multiple patterns `"(a)-[e1]->(_ac1);(_ac1)-[e1]->(c)"` by inserting interim vertexes arbitrarily. It specifies two edges from `a` to `_ac1` to `c`. +* Simply, you can also quantify the fixed length like `"(a)-[e*2]->(c)"`. The motif parser decompose it into multiple patterns `"(a)-[e1]->(_ac1);(_ac1)-[e1]->(c)"` by inserting interim vertices arbitrarily. It specifies two edges from `a` to `_ac1` to `c`. * In order to search for variable-length motifs, you can specify the range `"(a)-[e*1..3]->(c)"`. It unions the results from each possible length `"(a)-[e*1]->(c)"`, `"(a)-[e*2]->(c)"`, and `"(a)-[e*3]->(c)"` into a DataFrame. * If the direction is omitted `"(a)-[e]-(b)"`, it represents an undirected pattern — that is, either `"(a)-[e]->(b)"` or `"(a)<-[e]-(b)"`, which includes edges that are incoming or outgoing. * Within a pattern, names can be assigned to vertices and edges. For example, @@ -46,7 +46,7 @@ Restrictions: * Motifs are not allowed to contain edges without any named elements: `"()-[]->()"`, `"!()-[]->()"`, `"()-[]-()"`, and `"!()-[]-()"` are prohibited terms. * Motifs are not allowed to contain named edges within negated terms (since these named edges would never appear within results). E.g., `"!(a)-[ab]->(b)"` is invalid, but `"!(a)-[]->(b)"` is valid. * Negation is not supported for the variable-length pattern, bidirectional pattern and undirected pattern: `"!(a)-[*1..3]->(b)"`, `"!(a)<-[]->(b)"` and `"!(a)-[]-(b)"` are not allowed. -* Unbounded length patten is not supported: `"(a)-[*..3]->(b)"` and `"(a)-[*1..]->(b)"` are not allowed. +* Unbounded length pattern is not supported: `"(a)-[*..3]->(b)"` and `"(a)-[*1..]->(b)"` are not allowed. * You cannot join additional edges with a variable length pattern: `"(a)-[*1..3]->(b);(b)-[]->(c)"` is not allowed. More complex queries, such as queries which operate on vertex or edge attributes, diff --git a/docs/src/04-user-guide/05-traversals.md b/docs/src/04-user-guide/05-traversals.md index 479b2e212..d0d4f4aa9 100644 --- a/docs/src/04-user-guide/05-traversals.md +++ b/docs/src/04-user-guide/05-traversals.md @@ -57,11 +57,11 @@ For `graphframes` only. To avoid exponential growing of the Spark' Logical Plan, - `use_local_checkpoints` -For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are realiable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. +For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are reliable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. - `storage_level` -The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarious, using `DISK_ONLY` may be faster. +The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarios, using `DISK_ONLY` may be faster. - `is_direted` @@ -533,11 +533,11 @@ For `graphframes` only. To avoid exponential growing of the Spark' Logical Plan, - `use_local_checkpoints` -For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are realiable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. +For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are reliable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. - `storage_level` -The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarious, using `DISK_ONLY` may be faster. +The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarios, using `DISK_ONLY` may be faster. ## Aggregate Neighbors diff --git a/docs/src/04-user-guide/06-graph-clustering.md b/docs/src/04-user-guide/06-graph-clustering.md index b8b0dafd9..439b75b78 100644 --- a/docs/src/04-user-guide/06-graph-clustering.md +++ b/docs/src/04-user-guide/06-graph-clustering.md @@ -44,7 +44,7 @@ result.select("id", "label").show() - `maxIter` -An amount of Pregel iterations. While in theory, Label Propagation algorithm should converge sooner or later to some stable state, there are a lot of problems with it on a real-world graphs. The first one is oscillations: even if the algorithm is almost converged, on a big graphs some vertices at the border between detected communities may contibue oscilate from one iteration to another. The biggest problme, however, is that algorithm may easily converge to the state when all vertices has the same label. It is strongly recommended to set `maxIter` to some reasonable value from `5` to `10` and do some experiments depends of the task and the goal. +An amount of Pregel iterations. While in theory, Label Propagation algorithm should converge sooner or later to some stable state, there are a lot of problems with it on a real-world graphs. The first one is oscillations: even if the algorithm is almost converged, on a big graphs some vertices at the border between detected communities may contibue oscillate from one iteration to another. The biggest problem, however, is that algorithm may easily converge to the state when all vertices has the same label. It is strongly recommended to set `maxIter` to some reasonable value from `5` to `10` and do some experiments depends of the task and the goal. - `algorithm` @@ -56,11 +56,11 @@ For `graphframes` only. To avoid exponential growing of the Spark' Logical Plan, - `use_local_checkpoints` -For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are realiable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. +For `graphframes` only. By default, GraphFrames uses persistent checkpoints. They are reliable and reduce the errors rate. The downside of the persistent checkpoints is that they are requiride to set up a `checkpointDir` in persistent storage like `S3` or `HDFS`. By providing `use_local_checkpoints=True`, user can say GraphFrames to use local disks of Spark' executurs for checkpointing. Local checkpoints are faster, but they are less reliable: if the executur lost, for example, is taking by the higher priority job, checkpoints will be lost and the whole job fails. - `storage_level` -The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarious, using `DISK_ONLY` may be faster. +The level of storage for intermediate results and the output `DataFrame` with components. By default it is memory and disk deserialized as a good balance between performance and reliability. For very big graphs and out-of-core scenarios, using `DISK_ONLY` may be faster. ## Neighborhood-Aware CDLP @@ -213,7 +213,7 @@ An MIS is a set of vertices such that no two vertices in the set are adjacent (i The algorithm implemented in GraphFrames is based on the paper: Ghaffari, Mohsen. "An improved distributed algorithm for maximal independent set." Proceedings of the twenty-seventh annual ACM-SIAM symposium on Discrete algorithms. Society for Industrial and Applied Mathematics, 2016. ([https://doi.org/10.1137/1.9781611974331.ch20](https://doi.org/10.1137/1.9781611974331.ch20)) -Algorithm is useful, for example, if you are planning a marketing campaign and want to cover at most of users of the social network by minimizing communications. In that case, you need to find a big subset of nodes does not connected to each other but connected to othe nodes of the network. It is exactly what Maximal Independent Set do. It returns a set of not connected vertices so no more vertices can be added to the set without violation of the independence. +Algorithm is useful, for example, if you are planning a marketing campaign and want to cover at most of users of the social network by minimizing communications. In that case, you need to find a big subset of nodes does not connected to each other but connected to other nodes of the network. It is exactly what Maximal Independent Set do. It returns a set of not connected vertices so no more vertices can be added to the set without violation of the independence. Note: This is a randomized, non-deterministic algorithm. The result may vary between runs even if a fixed random seed is provided because how Apache Spark works. diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md index 8b673377c..c969cc601 100644 --- a/docs/src/05-blog/996-graphframes-012-release.md +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -24,7 +24,7 @@ After introducing the `AggregateNeighbors` API in version `0.11.0`, which is a g Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). -## Aproximate Neighbor Functions +## Approximate Neighbor Functions This release brings a foundation API for the approximate neighbor functions. Users can use it to cpmoute an approximate graph diameter, HyperBALL or approximate closeness centrality. diff --git a/docs/src/05-blog/998-graphframes-010-release.md b/docs/src/05-blog/998-graphframes-010-release.md index 3c8a957da..a093c75c3 100644 --- a/docs/src/05-blog/998-graphframes-010-release.md +++ b/docs/src/05-blog/998-graphframes-010-release.md @@ -30,13 +30,13 @@ The problem here is when we are folding the messages `RDD`, the computational co Because GraphX was deprecated in Apache Spark starting from the version `4.0` and does not accept patches anymore, GraphFrame maintainers made a decision to create an internal fork of GraphX. The fist change in the new fork was an improvement in the LabelPropagation. Instead of sending `Map[VertexID, Label]` the new implementation send `Vector[Label]`. In that case, the reduce step has a complexity *O(N)* because concatenations of `Vector` in Scala has a constant complexity. And the final reduction to compute the most common label is done on the step of label updating. While collecting the whole vector of labels may increase the average memory consumption, it reduce the peak memory consumption of algorithm. On the first iteration of LabelPropagation all the labels are unique, so old implementation will materialize the `Map[VertexID, Label]` with a size equal to amount of neighbors. At the same time, the new implementation will materialize only the `Vector[Label]` with the same size. Based on [benchmarking of Scala collections](https://www.lihaoyi.com/post/BenchmarkingScalaCollections.html#memory-use-of-immutable-collections), the memory consumption of the `Vector` is around 5 times less compared to the `Map`. -The result of the new implementation is around 70x boost: on a LDBC' Wiki-talk test graph (2M vertices, 5M edges) Spark' implementation runs in ~3500 seconds wile the new one runs in ~50 seconds. +The result of the new implementation is around 70x boost: on a LDBC' Wiki-talk test graph (2M vertices, 5M edges) Spark' implementation runs in ~3500 seconds while the new one runs in ~50 seconds. ### GraphX memory management Because main structures in GraphFrames are `DataFrame` objects but GraphX operates on `EdgeRDD` and `VertexRDD`, all the algorithm from GraphX are accessible from GraphFrames via conversion. Spark GraphX obviously was not designed to this way of usage. Under the hood GraphX do a lot of `RDD.persist` operations. For example, creating `Graph` from edges returns a persistent graph as well as result of all the graph algorithms are persistent. But on the GraphFrame side it is hard to unpersist all the intermediate RDDs. That tends to memory leaks. While it may be not a big problem in batch jobs, calling any GraphX algorithm in Spark Structured Streaming will lead to OOM errors after around 30-50 iterations. -With a full control of the GraphX fork, the team of GraphFrame maintainers was able to resolve the problem by removing from the GraphX code overpersisting as well, as handling manual unpersisting of GraphX structures after conversion. The result is no more memory leaks without loosing in performance. +With a full control of the GraphX fork, the team of GraphFrame maintainers was able to resolve the problem by removing from the GraphX code overpersisting as well, as handling manual unpersisting of GraphX structures after conversion. The result is no more memory leaks without losing in performance. ### Connected Components & AQE diff --git a/docs/src/06-contributing/01-contributing-guide.md b/docs/src/06-contributing/01-contributing-guide.md index 2d7d18072..dcf6a5295 100644 --- a/docs/src/06-contributing/01-contributing-guide.md +++ b/docs/src/06-contributing/01-contributing-guide.md @@ -224,7 +224,7 @@ PySpark session. ### 4.2 PySpark Connect update -PySpark Connect Plugin messages are located in `connect/src/main/protobuf`. After making any changes to messages, for exampple, after adding a new API the following is required: +PySpark Connect Plugin messages are located in `connect/src/main/protobuf`. After making any changes to messages, for example, after adding a new API the following is required: - re-compile the connect project that will trigger generation of new Java classes: `./build/sbt connect/compile` - re-generate Python classes from protobuf via `buf`: `buf generate` @@ -296,7 +296,7 @@ An example is: A full list of built-in directives may be found in [Laika Documentation](https://typelevel.org/Laika/latest/07-reference/01-standard-directives.html). -### 7.2 Build and preivew +### 7.2 Build and preview To build documentation and run a preview server run `./build/sbt docs/laikaPreview`. diff --git a/docs/src/llms.txt b/docs/src/llms.txt index 3d35397a9..07b2916c9 100644 --- a/docs/src/llms.txt +++ b/docs/src/llms.txt @@ -65,7 +65,7 @@ GraphFrames is a package for Apache Spark that provides DataFrame-based graphs. 5. **Graph Clustering**: Label propagation and power iteration clustering algorithms 6. **Custom Algorithms**: Framework for implementing domain-specific graph algorithms 7. **Network Stability**: K-Core algorithm for finding a critical cores -8. **Marketing Compaigns**: Finding a set of vertices with big coverage via Maximal Independent Set +8. **Marketing Campaigns**: Finding a set of vertices with big coverage via Maximal Independent Set ## Supported Algorithms diff --git a/python/dev/build_jar.py b/python/dev/build_jar.py index 1d8c961c9..126f5254a 100644 --- a/python/dev/build_jar.py +++ b/python/dev/build_jar.py @@ -11,9 +11,7 @@ def build(spark_versions: Sequence[str] = ["4.0.2"]): assert spark_version[:3] in {"3.5", "4.0", "4.1"}, "Unsupported spark version!" project_root = Path(__file__).parent.parent.parent - sbt_executable = ( - project_root.joinpath("build").joinpath("sbt").absolute().__str__() - ) + sbt_executable = project_root.joinpath("build").joinpath("sbt").absolute().__str__() sbt_build_command = [ sbt_executable, f"-Dspark.version={spark_version}", diff --git a/python/docs/conf.py b/python/docs/conf.py index 27319097d..e189ce7ac 100644 --- a/python/docs/conf.py +++ b/python/docs/conf.py @@ -307,7 +307,7 @@ # The format is a list of tuples containing the path and title. #epub_pre_files = [] -# HTML files shat should be inserted after the pages created by sphinx. +# HTML files that should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. #epub_post_files = [] diff --git a/python/graphframes/examples/graphs.py b/python/graphframes/examples/graphs.py index 0a3af2028..416e1fd9c 100644 --- a/python/graphframes/examples/graphs.py +++ b/python/graphframes/examples/graphs.py @@ -97,7 +97,7 @@ def gridIsingModel(self, n: int, vStd: float = 1.0, eStd: float = 1.0) -> GraphF "Grid graph must have size >= 1, but was given invalid value n = {}".format(n) ) - # create coodinates grid + # create coordinates grid coordinates = self._spark.createDataFrame( itertools.product(range(n), range(n)), schema=("i", "j") ) diff --git a/python/graphframes/graphframe.py b/python/graphframes/graphframe.py index 447c63b7a..667058c19 100644 --- a/python/graphframes/graphframe.py +++ b/python/graphframes/graphframe.py @@ -847,7 +847,7 @@ def labelPropagation( See Scala documentation for more details. :param maxIter: the number of iterations to be performed - :param algorithm: implementation to use, posible values are "graphframes" and "graphx"; + :param algorithm: implementation to use, possible values are "graphframes" and "graphx"; "graphx" is faster for small-medium sized graphs, "graphframes" requires less amount of memory :param use_local_checkpoints: should local checkpoints be used, default false; @@ -1005,7 +1005,7 @@ def shortestPaths( See Scala documentation for more details. :param landmarks: a set of one or more landmarks - :param algorithm: implementation to use, posible values are "graphframes" and "graphx"; + :param algorithm: implementation to use, possible values are "graphframes" and "graphx"; "graphx" is faster for small-medium sized graphs, "graphframes" requires less amount of memory :param use_local_checkpoints: should local checkpoints be used, default false; From 78f5066a101fabf0d5a6732a9c75e0c7bf82edf1 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Tue, 16 Jun 2026 11:13:57 -0700 Subject: [PATCH 2/3] ci: pass explicit config to black/isort so root dev/ does not break discovery When ../dev is included, black/isort resolve their config from the common parent of all paths (the repo root, which has no pyproject.toml) and fall back to defaults. Pass --config/--settings-path explicitly. --- .github/workflows/python-ci.yml | 4 ++-- .pre-commit-config.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 0f88f6028..ef65e13b5 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -57,9 +57,9 @@ jobs: - name: Code style working-directory: ./python run: | - poetry run python -m black --check graphframes dev ../dev + poetry run python -m black --check --config pyproject.toml graphframes dev ../dev poetry run python -m flake8 graphframes dev ../dev - poetry run python -m isort --check graphframes dev ../dev + poetry run python -m isort --check --settings-path pyproject.toml graphframes dev ../dev - name: Build jar working-directory: ./python run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6163616d..41cb38981 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: hooks: - id: black name: black - entry: bash -c 'cd python && poetry run black graphframes/ dev/ ../dev/' -- + entry: bash -c 'cd python && poetry run black --config pyproject.toml graphframes/ dev/ ../dev/' -- language: system types: [python] @@ -15,7 +15,7 @@ repos: - id: isort name: isort - entry: bash -c 'cd python && poetry run isort graphframes/ dev/ ../dev/' -- + entry: bash -c 'cd python && poetry run isort --settings-path pyproject.toml graphframes/ dev/ ../dev/' -- language: system types: [python] From d3ec10350091f86f21a50d0401dbf7bb8727e477 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Tue, 16 Jun 2026 15:00:57 -0700 Subject: [PATCH 3/3] ci: format dev connect scripts to satisfy extended linting run_connect.py and stop_connect.py were added in #863 before the linting scope was extended to dev/. Apply black/isort and add a noqa for an unavoidable long config string so python-ci passes. --- python/dev/run_connect.py | 16 +++++++--------- python/dev/stop_connect.py | 4 +--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/python/dev/run_connect.py b/python/dev/run_connect.py index 5ab70997d..2d52b34fe 100644 --- a/python/dev/run_connect.py +++ b/python/dev/run_connect.py @@ -1,13 +1,12 @@ #!/usr/bin/python +import argparse import os import shutil import subprocess import sys from pathlib import Path -import argparse - SPARK_ARCHIVE_LINK = "https://dlcdn.apache.org/spark/spark-{}/spark-{}-bin-hadoop3-connect.tgz" @@ -18,7 +17,7 @@ spark = str(args.spark) spark_full_link = SPARK_ARCHIVE_LINK.format(spark, spark) - + prj_root = Path(__file__).parent.parent.parent print("Build Graphframes...") @@ -47,7 +46,7 @@ spark_archive = spark_full_link.split("/")[-1] unpacked_spark_binary = spark_archive[:-4] - + if not tmp_dir.joinpath(unpacked_spark_binary).exists(): print(f"Downloading spark {spark}...") if tmp_dir.joinpath(spark_archive).exists(): @@ -82,8 +81,8 @@ "-xzf", spark_archive, ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True, ) if unpack_spark.returncode == 0: @@ -127,7 +126,6 @@ if core_jar is None: raise ValueError("failed to locate core JAR") - _ = shutil.copyfile(core_jar, spark_home.joinpath(core_jar.name)) _ = shutil.copyfile(graphx_jar, spark_home.joinpath(graphx_jar.name)) _ = shutil.copyfile(connect_jar, spark_home.joinpath(connect_jar.name)) @@ -142,7 +140,7 @@ "--jars", f"{core_jar.name},{graphx_jar.name},{connect_jar.name}", "--conf", - "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect", + "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect", # noqa: E501 "--conf", "spark.checkpoint.dir=/tmp/GFTestsCheckpointDir", "--conf", @@ -150,7 +148,7 @@ "--conf", "spark.sql.shuffle.partitions=4", ] - + print("Starting SparkConnect Server...") spark_connect = subprocess.run( run_connect_command, diff --git a/python/dev/stop_connect.py b/python/dev/stop_connect.py index fb8a5e491..3b9b22d82 100644 --- a/python/dev/stop_connect.py +++ b/python/dev/stop_connect.py @@ -1,15 +1,13 @@ #!/usr/bin/python +import argparse import os import shutil import subprocess import sys from pathlib import Path -import argparse - - if __name__ == "__main__": parser = argparse.ArgumentParser() _ = parser.add_argument("spark", type=str, default="4.1.2")