Skip to content

feat: run tests agains spark 4.1 #787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SemyonSinchenko merged 4 commits into from
Jan 28, 2026
Merged

feat: run tests agains spark 4.1 #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0a22473
run tests agains spark 4.1
SemyonSinchenko Jan 28, 2026
bd4a7ec
update sbt plugins
SemyonSinchenko Jan 28, 2026
823ecdb
fix build_jar
SemyonSinchenko Jan 28, 2026
234deb0
specify python version for 4.1.x
SemyonSinchenko Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/docs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,18 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6
with:
fetch-depth: 0

- name: Setup Java
uses: actions/setup-java@v4
uses: actions/setup-java@v5
with:
distribution: "zulu"
java-version: "17"

- name: Setup Python
uses: actions/setup-python@v4
uses: actions/setup-python@v6

- name: Install and configure Poetry
uses: snok/install-poetry@v1
Expand Down Expand Up @@ -62,7 +62,7 @@ jobs:
./build/sbt -Dspark.version=4.0.0 -Ddocs.mode=production "docs/laikaHTML"

- name: Upload artifact
uses: actions/upload-pages-artifact@v3
uses: actions/upload-pages-artifact@v6
with:
path: ./docs/target/docs/site

Expand Down
16 changes: 10 additions & 6 deletions .github/workflows/python-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,17 @@ jobs:
fail-fast: false
matrix:
include:
- spark-version: 3.5.6
- spark-version: 3.5.7
scala-version: 2.12
python-version: "3.10"
java-version: 11
- spark-version: 4.0.1
scala-version: 2.13
python-version: 3.12
python-version: "3.12"
java-version: 17
- spark-version: 4.1.0
scala-version: 2.13
python-version: "3.13"
java-version: 17
runs-on: ubuntu-latest
env:
Expand All @@ -23,17 +27,17 @@ jobs:
SCALA_VERSION: ${{ matrix.scala-version }}
PIP_REQUESTS_TIMEOUT: 100
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: "zulu"
java-version: "${{ matrix.java-version }}"
- uses: actions/cache@v4
- uses: actions/cache@v5
with:
path: |
~/.ivy2/cache
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
- uses: actions/setup-python@v4
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install and configure Poetry
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/python-publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,16 @@ jobs:
id-token: write
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up JDK
uses: actions/setup-java@v4
uses: actions/setup-java@v5
with:
java-version: '11'
distribution: 'zulu'

- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@v6
with:
python-version: '3.10'

Expand Down
13 changes: 8 additions & 5 deletions .github/workflows/scala-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,28 @@ jobs:
fail-fast: false
matrix:
include:
- spark-version: 3.5.6
- spark-version: 3.5.7
java-version: 11
- spark-version: 3.5.6
- spark-version: 3.5.7
java-version: 17
- spark-version: 4.0.1
java-version: 17
- spark-version: 4.1.0
java-version: 17

runs-on: ubuntu-latest
env:
# fixing this error after tests success: sbt.ForkMain failed with exit code 134
# https://stackoverflow.com/questions/33287424/strange-exception-in-sbt-test
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: "zulu"
java-version: "${{ matrix.java-version }}"
- uses: actions/cache@v4
- uses: actions/cache@v5
with:
path: |
~/.ivy2/cache
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/scala-publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
uses: actions/checkout@v6
with:
fetch-depth: 0 # sbt-git requires the full history to determine the version

- name: Set up JDK
uses: actions/setup-java@v4
uses: actions/setup-java@v5
with:
java-version: "${{ matrix.java-version }}"
distribution: 'zulu'
Expand Down
8 changes: 4 additions & 4 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ import laika.config.SyntaxHighlighting
import laika.format.Markdown.GitHubFlavor
import org.typelevel.scalacoptions.ScalacOptions

lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.6")
lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.7")
lazy val sparkMajorVer = sparkVer.substring(0, 1)
lazy val sparkBranch = sparkVer.substring(0, 3)
lazy val scalaVersions = sparkMajorVer match {
case "4" => Seq("2.13.16")
case "3" => Seq("2.12.20", "2.13.16")
case "4" => Seq("2.13.18")
case "3" => Seq("2.12.21", "2.13.18")
case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
}
lazy val scalaVer = sys.props.getOrElse("scala.version", scalaVersions.head)
Expand Down Expand Up @@ -65,7 +65,7 @@ ThisBuild / crossScalaVersions := scalaVersions

// Scalafix configuration
ThisBuild / semanticdbEnabled := true
ThisBuild / semanticdbVersion := "4.12.3" // The maximal version that supports both 2.13.12 and 2.12.18
ThisBuild / semanticdbVersion := "4.14.5"

// Don't publish the root project
publishArtifact := false
Expand Down
16 changes: 8 additions & 8 deletions project/plugins.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@ ThisBuild / libraryDependencySchemes ++= Seq(
"org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
)

addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.4.4")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.5")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.6")

// Protobuf things needed for the Spark Connect
addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.7")
addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.8")
libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.10.10"

// Scalafix
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3")
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.5")

// SBT CI Release
addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1")
addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2")

// JMH & benchmarking
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7")
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.8")

// Laika
addSbtPlugin("org.typelevel" % "laika-sbt" % "1.3.2")
addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.7.2")
addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.8.2")

// Typelevel helper
addSbtPlugin("org.typelevel" % "sbt-tpolecat" % "0.5.2")
Expand All @@ -36,4 +36,4 @@ libraryDependencies ++= Seq(
"io.circe" %% "circe-core",
"io.circe" %% "circe-generic",
"io.circe" %% "circe-parser"
).map(_ % circeVersion)
).map(_ % circeVersion)
2 changes: 1 addition & 1 deletion python/dev/build_jar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ def build(spark_versions: Sequence[str] = ["4.0.1"]):
for spark_version in spark_versions:
print("Building GraphFrames JAR...")
print(f"SPARK_VERSION: {spark_version[:3]}")
assert spark_version[:3] in {"3.5", "4.0"}, "Unsupported spark version!"
assert spark_version[:3] in {"3.5", "4.0", "4.1"}, "Unsupported spark version!"

project_root = Path(__file__).parent.parent.parent
sbt_executable = (
Expand Down