diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 66d4b474b..9605bb4a8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,18 +17,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Setup Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: "zulu" java-version: "17" - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 - name: Install and configure Poetry uses: snok/install-poetry@v1 @@ -62,7 +62,7 @@ jobs: ./build/sbt -Dspark.version=4.0.0 -Ddocs.mode=production "docs/laikaHTML" - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v6 with: path: ./docs/target/docs/site diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 62aa5befb..f1a827070 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -6,13 +6,17 @@ jobs: fail-fast: false matrix: include: - - spark-version: 3.5.6 + - spark-version: 3.5.7 scala-version: 2.12 python-version: "3.10" java-version: 11 - spark-version: 4.0.1 scala-version: 2.13 - python-version: 3.12 + python-version: "3.12" + java-version: 17 + - spark-version: 4.1.0 + scala-version: 2.13 + python-version: "3.13" java-version: 17 runs-on: ubuntu-latest env: @@ -23,17 +27,17 @@ jobs: SCALA_VERSION: ${{ matrix.scala-version }} PIP_REQUESTS_TIMEOUT: 100 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 with: distribution: "zulu" java-version: "${{ matrix.java-version }}" - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.ivy2/cache key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install and configure Poetry diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 660096abc..41089bdf5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -19,16 +19,16 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '11' distribution: 'zulu' - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.10' diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 88464fff8..378963eb4 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -6,12 +6,15 @@ jobs: fail-fast: false matrix: include: - - spark-version: 3.5.6 + - spark-version: 3.5.7 java-version: 11 - - spark-version: 3.5.6 + - spark-version: 3.5.7 java-version: 17 - spark-version: 4.0.1 java-version: 17 + - spark-version: 4.1.0 + java-version: 17 + runs-on: ubuntu-latest env: # fixing this error after tests success: sbt.ForkMain failed with exit code 134 @@ -19,12 +22,12 @@ jobs: JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8 JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 with: distribution: "zulu" java-version: "${{ matrix.java-version }}" - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.ivy2/cache diff --git a/.github/workflows/scala-publish.yml b/.github/workflows/scala-publish.yml index a9b0547e1..c2354ec10 100644 --- a/.github/workflows/scala-publish.yml +++ b/.github/workflows/scala-publish.yml @@ -27,12 +27,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # sbt-git requires the full history to determine the version - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: "${{ matrix.java-version }}" distribution: 'zulu' diff --git a/build.sbt b/build.sbt index 13561823f..44039cae0 100644 --- a/build.sbt +++ b/build.sbt @@ -3,12 +3,12 @@ import laika.config.SyntaxHighlighting import laika.format.Markdown.GitHubFlavor import org.typelevel.scalacoptions.ScalacOptions -lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.6") +lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.7") lazy val sparkMajorVer = sparkVer.substring(0, 1) lazy val sparkBranch = sparkVer.substring(0, 3) lazy val scalaVersions = sparkMajorVer match { - case "4" => Seq("2.13.16") - case "3" => Seq("2.12.20", "2.13.16") + case "4" => Seq("2.13.18") + case "3" => Seq("2.12.21", "2.13.18") case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.") } lazy val scalaVer = sys.props.getOrElse("scala.version", scalaVersions.head) @@ -65,7 +65,7 @@ ThisBuild / crossScalaVersions := scalaVersions // Scalafix configuration ThisBuild / semanticdbEnabled := true -ThisBuild / semanticdbVersion := "4.12.3" // The maximal version that supports both 2.13.12 and 2.12.18 +ThisBuild / semanticdbVersion := "4.14.5" // Don't publish the root project publishArtifact := false diff --git a/project/plugins.sbt b/project/plugins.sbt index 47334f8d6..c1cec25d4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,26 +5,26 @@ ThisBuild / libraryDependencySchemes ++= Seq( "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always ) -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.4.4") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.5") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.6") // Protobuf things needed for the Spark Connect -addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.7") +addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.8") libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.10.10" // Scalafix -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.5") // SBT CI Release -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") // JMH & benchmarking -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.8") // Laika addSbtPlugin("org.typelevel" % "laika-sbt" % "1.3.2") -addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.7.2") +addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.8.2") // Typelevel helper addSbtPlugin("org.typelevel" % "sbt-tpolecat" % "0.5.2") @@ -36,4 +36,4 @@ libraryDependencies ++= Seq( "io.circe" %% "circe-core", "io.circe" %% "circe-generic", "io.circe" %% "circe-parser" -).map(_ % circeVersion) \ No newline at end of file +).map(_ % circeVersion) diff --git a/python/dev/build_jar.py b/python/dev/build_jar.py index 94f8bbc0e..f291f6f04 100644 --- a/python/dev/build_jar.py +++ b/python/dev/build_jar.py @@ -8,7 +8,7 @@ def build(spark_versions: Sequence[str] = ["4.0.1"]): for spark_version in spark_versions: print("Building GraphFrames JAR...") print(f"SPARK_VERSION: {spark_version[:3]}") - assert spark_version[:3] in {"3.5", "4.0"}, "Unsupported spark version!" + assert spark_version[:3] in {"3.5", "4.0", "4.1"}, "Unsupported spark version!" project_root = Path(__file__).parent.parent.parent sbt_executable = (