diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 032b46592..bee91ff43 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -9,7 +9,7 @@ jobs: - spark-version: 3.5.4 scala-version: 2.13.8 - spark-version: 3.5.4 - scala-version: 2.12.12 + scala-version: 2.12.18 runs-on: ubuntu-22.04 env: # fixing this error after tests success: sbt.ForkMain failed with exit code 134 diff --git a/.gitignore b/.gitignore index 3f992a889..a07973c1e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ lib_managed/ src_managed/ project/boot/ project/plugins/project/ +.bsp # intellij .idea/ diff --git a/build.sbt b/build.sbt index d49da406e..061901717 100644 --- a/build.sbt +++ b/build.sbt @@ -1,92 +1,93 @@ -// Your sbt build file. Guides on how to write one can be found at -// http://www.scala-sbt.org/0.13/docs/index.html - import ReleaseTransformations._ -val sparkVer = sys.props.getOrElse("spark.version", "3.5.3") -val sparkBranch = sparkVer.substring(0, 3) -val defaultScalaVer = sparkBranch match { +lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.4") +lazy val sparkBranch = sparkVer.substring(0, 3) +lazy val defaultScalaVer = sparkBranch match { case "3.5" => "2.12.18" case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.") } -val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer) -val defaultScalaTestVer = scalaVer match { +lazy val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer) +lazy val defaultScalaTestVer = scalaVer match { case s if s.startsWith("2.12") || s.startsWith("2.13") => "3.0.8" } -sparkVersion := sparkVer - -scalaVersion := scalaVer - -name := "graphframes" - -spName := "graphframes/graphframes" - -organization := "org.graphframes" - -version := (version in ThisBuild).value + s"-spark$sparkBranch" - -isSnapshot := version.value.contains("SNAPSHOT") - -// All Spark Packages need a license -licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")) - -spAppendScalaVersion := true - -// Add Spark components this package depends on, e.g, "mllib", .... -sparkComponents ++= Seq("graphx", "sql", "mllib") - -// uncomment and change the value below to change the directory where your zip artifact will be created -// spDistDirectory := target.value - -// add any Spark Package dependencies using spDependencies. -// e.g. spDependencies += "databricks/spark-avro:0.1" - -libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.16" - -libraryDependencies += "org.scalatest" %% "scalatest" % defaultScalaTestVer % "test" - -libraryDependencies += "com.github.zafarkhaja" % "java-semver" % "0.9.0" % "test" // MIT license - -parallelExecution := false - -scalacOptions ++= Seq("-deprecation", "-feature") - -scalacOptions in (Compile, doc) ++= Seq( - "-groups", - "-implicits", - "-skip-packages", Seq("org.apache.spark").mkString(":")) - -scalacOptions in (Test, doc) ++= Seq("-groups", "-implicits") - -// This fixes a class loader problem with scala.Tuple2 class, scala-2.11, Spark 2.x -fork in Test := true - -// This and the next line fix a problem with forked run: https://github.com/scalatest/scalatest/issues/770 -javaOptions in Test ++= Seq( - "-XX:+IgnoreUnrecognizedVMOptions", - "-Xmx2048m", - "-XX:ReservedCodeCacheSize=384m", - "-XX:MaxMetaspaceSize=384m", - "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", - "--add-opens=java.base/java.lang=ALL-UNNAMED" -) - -concurrentRestrictions in Global := Seq( - Tags.limitAll(1)) - -autoAPIMappings := true - -coverageHighlighting := false - -// We only use sbt-release to update version numbers. -releaseProcess := Seq[ReleaseStep]( - inquireVersions, - setReleaseVersion, - commitReleaseVersion, - tagRelease, - setNextVersion, - commitNextVersion -) +ThisBuild / version := { + val baseVersion = (ThisBuild / version).value + s"${baseVersion}-spark${sparkBranch}" +} -credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials") +ThisBuild / scalaVersion := scalaVer +ThisBuild / organization := "org.graphframes" +ThisBuild / crossScalaVersions := Seq("2.12.18", "2.13.8") + +lazy val root = (project in file(".")) + .settings( + name := "graphframes", + + // Replace spark-packages plugin functionality with explicit dependencies + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-graphx" % sparkVer % "provided" cross CrossVersion.for3Use2_13, + "org.apache.spark" %% "spark-sql" % sparkVer % "provided" cross CrossVersion.for3Use2_13, + "org.apache.spark" %% "spark-mllib" % sparkVer % "provided" cross CrossVersion.for3Use2_13, + "org.slf4j" % "slf4j-api" % "1.7.16", + "org.scalatest" %% "scalatest" % defaultScalaTestVer % Test, + "com.github.zafarkhaja" % "java-semver" % "0.9.0" % Test + ), + + licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")), + + // Modern way to set Scala options + Compile / scalacOptions ++= Seq("-deprecation", "-feature"), + + Compile / doc / scalacOptions ++= Seq( + "-groups", + "-implicits", + "-skip-packages", Seq("org.apache.spark").mkString(":") + ), + + Test / doc / scalacOptions ++= Seq("-groups", "-implicits"), + + // Test settings + Test / fork := true, + Test / parallelExecution := false, + + Test / javaOptions ++= Seq( + "-XX:+IgnoreUnrecognizedVMOptions", + "-Xmx2048m", + "-XX:ReservedCodeCacheSize=384m", + "-XX:MaxMetaspaceSize=384m", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/java.lang=ALL-UNNAMED" + ), + + // Global settings + Global / concurrentRestrictions := Seq( + Tags.limitAll(1) + ), + + autoAPIMappings := true, + + coverageHighlighting := false, + + // Release settings + releaseProcess := Seq[ReleaseStep]( + inquireVersions, + setReleaseVersion, + commitReleaseVersion, + tagRelease, + setNextVersion, + commitNextVersion + ), + + // Assembly settings + assembly / test := {}, // No tests in assembly + assembly / assemblyMergeStrategy := { + case PathList("META-INF", xs @ _*) => MergeStrategy.discard + case x if x.endsWith("module-info.class") => MergeStrategy.discard + case x => + val oldStrategy = (assembly / assemblyMergeStrategy).value + oldStrategy(x) + }, + + credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials") + ) \ No newline at end of file diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 6ab8d7313..01ba6b929 100755 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -7,7 +7,7 @@ # TODO - Should we merge the main SBT script with this library? if test -z "$HOME"; then - declare -r script_dir="$(dirname $script_path)" + declare -r script_dir="$(dirname "$script_path")" else declare -r script_dir="$HOME/.sbt" fi @@ -17,6 +17,7 @@ declare -a java_args declare -a scalac_args declare -a sbt_commands declare -a maven_profiles +declare sbt_default_mem=4096 if test -x "$JAVA_HOME/bin/java"; then echo -e "Using $JAVA_HOME as default JAVA_HOME." @@ -37,32 +38,36 @@ dlog () { } acquire_sbt_jar () { - SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties` - URL1=https://repo.scala-sbt.org/scalasbt/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar + SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` + # DEFAULT_ARTIFACT_REPOSITORY env variable can be used to only fetch + # artifacts from internal repos only. + # Ex: + # DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/ + URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR if [[ ! -f "$sbt_jar" ]]; then # Download sbt launch jar if it hasn't been downloaded yet - if [ ! -f ${JAR} ]; then + if [ ! -f "${JAR}" ]; then # Download printf "Attempting to fetch sbt\n" - JAR_DL=${JAR}.part - if hash curl 2>/dev/null; then + JAR_DL="${JAR}.part" + if [ $(command -v curl) ]; then curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" - elif hash wget 2>/dev/null; then + elif [ $(command -v wget) ]; then wget --quiet ${URL1} -O "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" else - printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" + printf "You do not have curl or wget installed, please install sbt manually from https://www.scala-sbt.org/\n" exit -1 fi fi - if [ ! -f ${JAR} ]; then + if [ ! -f "${JAR}" ]; then # We failed to download - printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n" + printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from https://www.scala-sbt.org/\n" exit -1 fi printf "Launching sbt from ${JAR}\n" @@ -82,7 +87,7 @@ execRunner () { echo "" } - exec "$@" + "$@" } addJava () { @@ -105,19 +110,18 @@ addResidual () { residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { - addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" + addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert # so they need not be dicked around with individually. get_mem_opts () { - local mem=${1:-2048} - local perm=$(( $mem / 4 )) - (( $perm > 256 )) || perm=256 - (( $perm < 4096 )) || perm=4096 - local codecache=$(( $perm / 2 )) + local mem=${1:-$sbt_default_mem} + local codecache=$(( $mem / 8 )) + (( $codecache > 128 )) || codecache=128 + (( $codecache < 2048 )) || codecache=2048 - echo "-Xms${mem}m -Xmx${mem}m -XX:MaxMetaspaceSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m" + echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" } require_arg () { @@ -125,7 +129,8 @@ require_arg () { local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then - die "$opt requires <$type> argument" + echo "$opt requires <$type> argument" 1>&2 + exit 1 fi } @@ -178,18 +183,11 @@ run() { # run sbt execRunner "$java_cmd" \ - ${SBT_OPTS:-$default_sbt_opts} \ $(get_mem_opts $sbt_mem) \ + ${SBT_OPTS:-$default_sbt_opts} \ ${java_opts} \ ${java_args[@]} \ -jar "$sbt_jar" \ "${sbt_commands[@]}" \ "${residual_args[@]}" } - -runAlternateBoot() { - local bootpropsfile="$1" - shift - addJava "-Dsbt.boot.properties=$bootpropsfile" - run $@ -} diff --git a/project/build.properties b/project/build.properties index 5c4bcd91b..3c3220437 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ // This file should only contain the version of sbt to use. -sbt.version=0.13.18 +sbt.version=1.9.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index a27f399b4..46028c336 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,10 +1,10 @@ -// You may use this file to add plugin dependencies for sbt. -resolvers += "Spark Packages repo" at "https://repos.spark-packages.org/" - -addSbtPlugin("org.spark-packages" %% "sbt-spark-package" % "0.2.6") - -// scalacOptions in (Compile,doc) := Seq("-groups", "-implicits") - -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") - -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") +// https://github.com/scala/bug/issues/12632 +// https://github.com/scoverage/sbt-scoverage/issues/475 +// Workaround: +ThisBuild / libraryDependencySchemes ++= Seq( + "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always +) + +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.10") +addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") diff --git a/version.sbt b/version.sbt index ca68fcbd5..f72bdcc0e 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.8.4" +ThisBuild / version := "0.8.4"