From ebb91daf0f4b0e8d028c95127e29771b579038ba Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 11 Jun 2026 22:24:24 +0200 Subject: [PATCH 1/5] feat: 012 release blogpost + update the ci matrix --- .github/workflows/docs.yml | 4 +- .github/workflows/python-ci.yml | 6 +-- .github/workflows/scala-ci.yml | 10 ++--- docs/src/05-blog/01-index.md | 1 + .../05-blog/996-graphframes-012-release.md | 38 +++++++++++++++++++ docs/src/05-blog/directory.conf | 1 + 6 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 docs/src/05-blog/996-graphframes-012-release.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f1709a440..dacdeb2ef 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -55,11 +55,11 @@ jobs: - name: Run benchmarks run: | - python ./dev/run_doc_benchmarks.py --spark-version 4.0.0 + python ./dev/run_doc_benchmarks.py --spark-version 4.1.2 - name: Build with Laika run: | - ./build/sbt -Dspark.version=4.0.0 -Ddocs.mode=production "docs/laikaHTML" + ./build/sbt -Dspark.version=4.1.2 -Ddocs.mode=production "docs/laikaHTML" - name: Upload artifact uses: actions/upload-pages-artifact@v4 diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1ebcfc467..40ad81a5d 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -8,15 +8,15 @@ jobs: fail-fast: false matrix: include: - - spark-version: 3.5.7 + - spark-version: 3.5.8 scala-version: 2.12 python-version: "3.10" java-version: 11 - - spark-version: 4.0.1 + - spark-version: 4.0.3 scala-version: 2.13 python-version: "3.12" java-version: 17 - - spark-version: 4.1.0 + - spark-version: 4.1.2 scala-version: 2.13 python-version: "3.13" java-version: 17 diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index a5c0a00fe..a3ed59b69 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -8,15 +8,15 @@ jobs: fail-fast: false matrix: include: - - spark-version: 3.5.7 + - spark-version: 3.5.8 java-version: 8 - - spark-version: 3.5.7 + - spark-version: 3.5.8 java-version: 11 - - spark-version: 3.5.7 + - spark-version: 3.5.8 java-version: 17 - - spark-version: 4.0.1 + - spark-version: 4.0.3 java-version: 17 - - spark-version: 4.1.0 + - spark-version: 4.1.2 java-version: 17 runs-on: ubuntu-latest diff --git a/docs/src/05-blog/01-index.md b/docs/src/05-blog/01-index.md index acdb919f8..eacd3641e 100644 --- a/docs/src/05-blog/01-index.md +++ b/docs/src/05-blog/01-index.md @@ -6,6 +6,7 @@ It is an official blog of the GraphFrames project. ## All posts: +- [GraphFrames 0.12.0 release](996-graphframes-012-release.md) - [GraphFrames 0.11.0 release](997-graphframes-011-release.md) - [GraphFrames 0.10.0 release](998-graphframes-010-release.md) - [GraphFrames 0.9.3 release](999-graphframes-093-release.md) diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md new file mode 100644 index 000000000..7a8071be0 --- /dev/null +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -0,0 +1,38 @@ +# GraphFrames 0.12.0 release + +- **Published:** 2026-06-12T00:00:00Z +- **Title:** GraphFrames 0.12.0 release +- **Summary:** This release brings new Community Detection algorithm, new API to find all simple paths between subset of vertices and significant performance improvements for the Two-Phase Connected Components Algorithm. + +## New Contributors + +- [@slavlotski](https://github.com/slavlotski) -- `asReversed` helper API to reverse all the edges of the graph + +## New Community Detection Algorithm + +Previous versions of GraphFrames relied entirely on the most naive implementation of the Label Propagation algorithm. While this implementation is fast and well-known, the quality of the output clusters is questionable, and the algorithm itself is unstable. Even small changes in the local structure can alter the output. + +The new algorithm significantly modifies the original Label Propagation algorithm. While it follows the same idea that allows for efficient implementation on distributed graphs, it also provides more flexibility. The inspiration came from [Xie, Jierui, and Boleslaw K. Szymanski. "Community detection using a neighborhood strength driven label propagation algorithm." 2011 IEEE Network Science Workshop. IEEE, 2011.](https://arxiv.org/abs/1105.3264) + +The core idea is that, during propagation, vertices choose a community based not only on their local neighborhood, but also on the number of neighbors they have in common with other community members. Compared to existing label propagation, the new algorithm also supports passing initial labels, which allows it to be used incrementally or for semi-supervised community detection. + +Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). + +## New all paths API + +After introducing the `AggregateNeighbors` API in version `0.11.0`, which is a generic, multi-hop aggregation API, GraphFrames is receiving built-in implementations based on neighbor aggregation. The first is the long-awaited API that finds all simple paths between a subset of vertices. + +Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). + +## Performance optimizations in Connected Components + +The Two-Phase algorithm is based on the idea of rewiring edges to end up with a star-like graph structure. However, during the rewiring process, a large number of leaves, or vertices with no outgoing edges, appear. Although determining components for these vertices is trivial, and they do not participate in the main algorithm loop, they still shuffle and join until full convergence. The new optimization adds an efficient way to determine the optimal time to remove such leaves and offset the cost of rejoining them after convergence. Based on initial benchmarks, the optimization delivers a ~25% performance boost. + +Credits to [WeichenXu123](https://github.com/WeichenXu123) + +## Future steps + +- Moving in the direction of support of full-featured graph queries +- Improving GraphFrames capabilities in Graph ML +- Adding features useful in Spatial Graphs analysis + diff --git a/docs/src/05-blog/directory.conf b/docs/src/05-blog/directory.conf index 28688c348..50b6688c1 100644 --- a/docs/src/05-blog/directory.conf +++ b/docs/src/05-blog/directory.conf @@ -2,6 +2,7 @@ laika.title = GraphFrames Blog laika.navigationOrder = [ 01-index.md + 996-graphframes-012-release.md 997-graphframes-011-release.md 998-graphframes-010-release.md 999-graphframes-093-release.md From b97bdad38d1785e7c4e59897aad94dc63b438881 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 11 Jun 2026 23:33:12 +0200 Subject: [PATCH 2/5] feat: add a note about the pregel breaking change --- docs/src/05-blog/996-graphframes-012-release.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md index 7a8071be0..ff1e1f9b9 100644 --- a/docs/src/05-blog/996-graphframes-012-release.md +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -30,9 +30,12 @@ The Two-Phase algorithm is based on the idea of rewiring edges to end up with a Credits to [WeichenXu123](https://github.com/WeichenXu123) +## Important note + +Previous versions of Graphframes had an unspecified contract within the Pregel API regarding the handling of edge attributes. All edge attributes, including the IDs of the source (`src`) and destination (`dst`) vertices, were implicitly packed into a `StructType` and persisted. Although persisting was required for performance, it blocked the Catalyst optimizer from eliminating these columns if they were not used. This resulted in an almost twofold increase in peak memory load in all scenarios and was considered a bug. Starting with version 0.12.0, users who want to use edge attributes in the low-level Pregel should specify them explicitly using `requiredEdgeColumns(...)` in Scala or `required_edge_columns(...)` in the Python API. + ## Future steps - Moving in the direction of support of full-featured graph queries - Improving GraphFrames capabilities in Graph ML - Adding features useful in Spatial Graphs analysis - From c7422527f04bd5c6e566e54faae6651ac314f571 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Fri, 12 Jun 2026 00:08:47 +0200 Subject: [PATCH 3/5] feat: typo --- docs/src/05-blog/996-graphframes-012-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md index ff1e1f9b9..14d875fd2 100644 --- a/docs/src/05-blog/996-graphframes-012-release.md +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -28,7 +28,7 @@ Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). The Two-Phase algorithm is based on the idea of rewiring edges to end up with a star-like graph structure. However, during the rewiring process, a large number of leaves, or vertices with no outgoing edges, appear. Although determining components for these vertices is trivial, and they do not participate in the main algorithm loop, they still shuffle and join until full convergence. The new optimization adds an efficient way to determine the optimal time to remove such leaves and offset the cost of rejoining them after convergence. Based on initial benchmarks, the optimization delivers a ~25% performance boost. -Credits to [WeichenXu123](https://github.com/WeichenXu123) +Credits to [@WeichenXu123](https://github.com/WeichenXu123) ## Important note From f6912eb9279cc4132654556ab949ec5a1f7bc530 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Fri, 12 Jun 2026 12:29:01 +0200 Subject: [PATCH 4/5] feat: mention Databricks --- docs/src/05-blog/996-graphframes-012-release.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md index 14d875fd2..4b17cec76 100644 --- a/docs/src/05-blog/996-graphframes-012-release.md +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -28,7 +28,9 @@ Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). The Two-Phase algorithm is based on the idea of rewiring edges to end up with a star-like graph structure. However, during the rewiring process, a large number of leaves, or vertices with no outgoing edges, appear. Although determining components for these vertices is trivial, and they do not participate in the main algorithm loop, they still shuffle and join until full convergence. The new optimization adds an efficient way to determine the optimal time to remove such leaves and offset the cost of rejoining them after convergence. Based on initial benchmarks, the optimization delivers a ~25% performance boost. -Credits to [@WeichenXu123](https://github.com/WeichenXu123) +This optimization was part of the Databricks' internal fork of GraphFrames. It was donated to the open-source GraphFrames by the company. + +Credits to [@WeichenXu123](https://github.com/WeichenXu123) and [Databricks](https://www.databricks.com/) ## Important note From 9d6eedf107e86bd04d8594c6b382f7300d7f97e1 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Sat, 13 Jun 2026 14:01:43 +0200 Subject: [PATCH 5/5] feat: mention HyperANF in the post --- docs/src/05-blog/996-graphframes-012-release.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/src/05-blog/996-graphframes-012-release.md b/docs/src/05-blog/996-graphframes-012-release.md index 4b17cec76..8b673377c 100644 --- a/docs/src/05-blog/996-graphframes-012-release.md +++ b/docs/src/05-blog/996-graphframes-012-release.md @@ -2,7 +2,7 @@ - **Published:** 2026-06-12T00:00:00Z - **Title:** GraphFrames 0.12.0 release -- **Summary:** This release brings new Community Detection algorithm, new API to find all simple paths between subset of vertices and significant performance improvements for the Two-Phase Connected Components Algorithm. +- **Summary:** This release brings new Community Detection algorithm, new API to find all simple paths between subset of vertices, approximate neighbor functions and significant performance improvements for the Two-Phase Connected Components Algorithm. ## New Contributors @@ -24,6 +24,12 @@ After introducing the `AggregateNeighbors` API in version `0.11.0`, which is a g Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). +## Aproximate Neighbor Functions + +This release brings a foundation API for the approximate neighbor functions. Users can use it to cpmoute an approximate graph diameter, HyperBALL or approximate closeness centrality. + +Credits to [@SemyonSinchenko](https://github.com/SemyonSinchenko). + ## Performance optimizations in Connected Components The Two-Phase algorithm is based on the idea of rewiring edges to end up with a star-like graph structure. However, during the rewiring process, a large number of leaves, or vertices with no outgoing edges, appear. Although determining components for these vertices is trivial, and they do not participate in the main algorithm loop, they still shuffle and join until full convergence. The new optimization adds an efficient way to determine the optimal time to remove such leaves and offset the cost of rejoining them after convergence. Based on initial benchmarks, the optimization delivers a ~25% performance boost.