diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 7625d4eee..5c7a65cf5 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -30,9 +30,9 @@ jobs: ~/.ivy2/cache key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-java-${{ matrix.java-version }} - name: Check scalafmt - run: build/sbt -Dspark.version=${{ matrix.spark-version }} root/scalafmtCheckAll connect/scalafmtCheckAll + run: build/sbt -Dspark.version=${{ matrix.spark-version }} scalafmtCheckAll - name: Check scalastyle - run: build/sbt -Dspark.version=${{ matrix.spark-version }} root/scalafixAll connect/scalafixAll + run: build/sbt -Dspark.version=${{ matrix.spark-version }} "scalafixAll --check" - name: Build and Test - run: build/sbt -v -Dspark.version=${{ matrix.spark-version }} coverage +test coverageReport "project connect" coverage +test coverageReport + run: build/sbt -v -Dspark.version=${{ matrix.spark-version }} coverage +test coverageReport - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/scala-publish.yml b/.github/workflows/scala-publish.yml index 0881d3224..94305155a 100644 --- a/.github/workflows/scala-publish.yml +++ b/.github/workflows/scala-publish.yml @@ -21,7 +21,7 @@ jobs: matrix: include: - spark-version: 3.5.5 - java-version: 11 + java-version: 8 - spark-version: 4.0.0 java-version: 17 runs-on: ubuntu-latest @@ -38,7 +38,7 @@ jobs: distribution: 'zulu' - name: Run the release of core - run: build/sbt -Dspark.version=${{ matrix.spark-version }} +ci-release "project connect" +ci-release + run: build/sbt -Dspark.version=${{ matrix.spark-version }} ci-release env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} diff --git a/build.sbt b/build.sbt index 92368cd12..299125026 100644 --- a/build.sbt +++ b/build.sbt @@ -100,7 +100,7 @@ lazy val commonSetting = Seq( else "-Wunused:imports")) -lazy val root = (project in file(".")) +lazy val core = (project in file("core")) .settings( commonSetting, name := "graphframes", @@ -122,8 +122,8 @@ lazy val root = (project in file(".")) Compile / packageDoc / publishArtifact := true, Compile / packageSrc / publishArtifact := true) -lazy val connect = (project in file("graphframes-connect")) - .dependsOn(root) +lazy val connect = (project in file("connect")) + .dependsOn(core) .settings( name := s"graphframes-connect", moduleName := s"${name.value}-spark${sparkMajorVer}", diff --git a/graphframes-connect/src/main/protobuf/graphframes.proto b/connect/src/main/protobuf/graphframes.proto similarity index 100% rename from graphframes-connect/src/main/protobuf/graphframes.proto rename to connect/src/main/protobuf/graphframes.proto diff --git a/graphframes-connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala b/connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala similarity index 100% rename from graphframes-connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala rename to connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala diff --git a/graphframes-connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala b/connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala similarity index 99% rename from graphframes-connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala rename to connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala index b579ee1b1..01a376166 100644 --- a/graphframes-connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala +++ b/connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala @@ -1,14 +1,12 @@ package org.apache.spark.sql.graphframes -import java.util.Optional - -import org.graphframes.connect.proto.GraphFramesAPI - +import com.google.protobuf import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connect.planner.SparkConnectPlanner import org.apache.spark.sql.connect.plugin.RelationPlugin +import org.graphframes.connect.proto.GraphFramesAPI -import com.google.protobuf +import java.util.Optional class GraphFramesConnect extends RelationPlugin { override def transform( diff --git a/graphframes-connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala b/connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala similarity index 100% rename from graphframes-connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala rename to connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala diff --git a/src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala b/core/src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala similarity index 100% rename from src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala rename to core/src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala diff --git a/src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala b/core/src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala similarity index 100% rename from src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala rename to core/src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala diff --git a/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala b/core/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala similarity index 100% rename from src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala rename to core/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala diff --git a/src/main/scala/org/graphframes/GraphFrame.scala b/core/src/main/scala/org/graphframes/GraphFrame.scala similarity index 100% rename from src/main/scala/org/graphframes/GraphFrame.scala rename to core/src/main/scala/org/graphframes/GraphFrame.scala diff --git a/src/main/scala/org/graphframes/GraphFramePythonAPI.scala b/core/src/main/scala/org/graphframes/GraphFramePythonAPI.scala similarity index 100% rename from src/main/scala/org/graphframes/GraphFramePythonAPI.scala rename to core/src/main/scala/org/graphframes/GraphFramePythonAPI.scala diff --git a/src/main/scala/org/graphframes/Logging.scala b/core/src/main/scala/org/graphframes/Logging.scala similarity index 100% rename from src/main/scala/org/graphframes/Logging.scala rename to core/src/main/scala/org/graphframes/Logging.scala diff --git a/src/main/scala/org/graphframes/examples/BeliefPropagation.scala b/core/src/main/scala/org/graphframes/examples/BeliefPropagation.scala similarity index 100% rename from src/main/scala/org/graphframes/examples/BeliefPropagation.scala rename to core/src/main/scala/org/graphframes/examples/BeliefPropagation.scala diff --git a/src/main/scala/org/graphframes/examples/Graphs.scala b/core/src/main/scala/org/graphframes/examples/Graphs.scala similarity index 100% rename from src/main/scala/org/graphframes/examples/Graphs.scala rename to core/src/main/scala/org/graphframes/examples/Graphs.scala diff --git a/src/main/scala/org/graphframes/examples/LDBCUtils.scala b/core/src/main/scala/org/graphframes/examples/LDBCUtils.scala similarity index 100% rename from src/main/scala/org/graphframes/examples/LDBCUtils.scala rename to core/src/main/scala/org/graphframes/examples/LDBCUtils.scala diff --git a/src/main/scala/org/graphframes/exceptions.scala b/core/src/main/scala/org/graphframes/exceptions.scala similarity index 100% rename from src/main/scala/org/graphframes/exceptions.scala rename to core/src/main/scala/org/graphframes/exceptions.scala diff --git a/src/main/scala/org/graphframes/lib/AggregateMessages.scala b/core/src/main/scala/org/graphframes/lib/AggregateMessages.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/AggregateMessages.scala rename to core/src/main/scala/org/graphframes/lib/AggregateMessages.scala diff --git a/src/main/scala/org/graphframes/lib/BFS.scala b/core/src/main/scala/org/graphframes/lib/BFS.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/BFS.scala rename to core/src/main/scala/org/graphframes/lib/BFS.scala diff --git a/src/main/scala/org/graphframes/lib/ConnectedComponents.scala b/core/src/main/scala/org/graphframes/lib/ConnectedComponents.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/ConnectedComponents.scala rename to core/src/main/scala/org/graphframes/lib/ConnectedComponents.scala diff --git a/src/main/scala/org/graphframes/lib/GraphXConversions.scala b/core/src/main/scala/org/graphframes/lib/GraphXConversions.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/GraphXConversions.scala rename to core/src/main/scala/org/graphframes/lib/GraphXConversions.scala diff --git a/src/main/scala/org/graphframes/lib/LabelPropagation.scala b/core/src/main/scala/org/graphframes/lib/LabelPropagation.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/LabelPropagation.scala rename to core/src/main/scala/org/graphframes/lib/LabelPropagation.scala diff --git a/src/main/scala/org/graphframes/lib/PageRank.scala b/core/src/main/scala/org/graphframes/lib/PageRank.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/PageRank.scala rename to core/src/main/scala/org/graphframes/lib/PageRank.scala diff --git a/src/main/scala/org/graphframes/lib/ParallelPersonalizedPageRank.scala b/core/src/main/scala/org/graphframes/lib/ParallelPersonalizedPageRank.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/ParallelPersonalizedPageRank.scala rename to core/src/main/scala/org/graphframes/lib/ParallelPersonalizedPageRank.scala diff --git a/src/main/scala/org/graphframes/lib/Pregel.scala b/core/src/main/scala/org/graphframes/lib/Pregel.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/Pregel.scala rename to core/src/main/scala/org/graphframes/lib/Pregel.scala diff --git a/src/main/scala/org/graphframes/lib/SVDPlusPlus.scala b/core/src/main/scala/org/graphframes/lib/SVDPlusPlus.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/SVDPlusPlus.scala rename to core/src/main/scala/org/graphframes/lib/SVDPlusPlus.scala diff --git a/src/main/scala/org/graphframes/lib/ShortestPaths.scala b/core/src/main/scala/org/graphframes/lib/ShortestPaths.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/ShortestPaths.scala rename to core/src/main/scala/org/graphframes/lib/ShortestPaths.scala diff --git a/src/main/scala/org/graphframes/lib/StronglyConnectedComponents.scala b/core/src/main/scala/org/graphframes/lib/StronglyConnectedComponents.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/StronglyConnectedComponents.scala rename to core/src/main/scala/org/graphframes/lib/StronglyConnectedComponents.scala diff --git a/src/main/scala/org/graphframes/lib/TriangleCount.scala b/core/src/main/scala/org/graphframes/lib/TriangleCount.scala similarity index 100% rename from src/main/scala/org/graphframes/lib/TriangleCount.scala rename to core/src/main/scala/org/graphframes/lib/TriangleCount.scala diff --git a/src/main/scala/org/graphframes/mixins.scala b/core/src/main/scala/org/graphframes/mixins.scala similarity index 100% rename from src/main/scala/org/graphframes/mixins.scala rename to core/src/main/scala/org/graphframes/mixins.scala diff --git a/src/main/scala/org/graphframes/pattern/patterns.scala b/core/src/main/scala/org/graphframes/pattern/patterns.scala similarity index 100% rename from src/main/scala/org/graphframes/pattern/patterns.scala rename to core/src/main/scala/org/graphframes/pattern/patterns.scala diff --git a/src/test/resources/als-test.data b/core/src/test/resources/als-test.data similarity index 100% rename from src/test/resources/als-test.data rename to core/src/test/resources/als-test.data diff --git a/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties similarity index 100% rename from src/test/resources/log4j.properties rename to core/src/test/resources/log4j.properties diff --git a/src/test/resources/log4j2.properties b/core/src/test/resources/log4j2.properties similarity index 100% rename from src/test/resources/log4j2.properties rename to core/src/test/resources/log4j2.properties diff --git a/src/test/scala/org/graphframes/GraphFrameSuite.scala b/core/src/test/scala/org/graphframes/GraphFrameSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/GraphFrameSuite.scala rename to core/src/test/scala/org/graphframes/GraphFrameSuite.scala diff --git a/src/test/scala/org/graphframes/GraphFrameTestSparkContext.scala b/core/src/test/scala/org/graphframes/GraphFrameTestSparkContext.scala similarity index 100% rename from src/test/scala/org/graphframes/GraphFrameTestSparkContext.scala rename to core/src/test/scala/org/graphframes/GraphFrameTestSparkContext.scala diff --git a/src/test/scala/org/graphframes/PatternMatchSuite.scala b/core/src/test/scala/org/graphframes/PatternMatchSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/PatternMatchSuite.scala rename to core/src/test/scala/org/graphframes/PatternMatchSuite.scala diff --git a/src/test/scala/org/graphframes/SparkFunSuite.scala b/core/src/test/scala/org/graphframes/SparkFunSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/SparkFunSuite.scala rename to core/src/test/scala/org/graphframes/SparkFunSuite.scala diff --git a/src/test/scala/org/graphframes/TestUtils.scala b/core/src/test/scala/org/graphframes/TestUtils.scala similarity index 100% rename from src/test/scala/org/graphframes/TestUtils.scala rename to core/src/test/scala/org/graphframes/TestUtils.scala diff --git a/src/test/scala/org/graphframes/examples/BeliefPropagationSuite.scala b/core/src/test/scala/org/graphframes/examples/BeliefPropagationSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/examples/BeliefPropagationSuite.scala rename to core/src/test/scala/org/graphframes/examples/BeliefPropagationSuite.scala diff --git a/src/test/scala/org/graphframes/examples/GraphsSuite.scala b/core/src/test/scala/org/graphframes/examples/GraphsSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/examples/GraphsSuite.scala rename to core/src/test/scala/org/graphframes/examples/GraphsSuite.scala diff --git a/src/test/scala/org/graphframes/ldbc/TestLDBCCases.scala b/core/src/test/scala/org/graphframes/ldbc/TestLDBCCases.scala similarity index 100% rename from src/test/scala/org/graphframes/ldbc/TestLDBCCases.scala rename to core/src/test/scala/org/graphframes/ldbc/TestLDBCCases.scala diff --git a/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala b/core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala rename to core/src/test/scala/org/graphframes/lib/AggregateMessagesSuite.scala diff --git a/src/test/scala/org/graphframes/lib/BFSSuite.scala b/core/src/test/scala/org/graphframes/lib/BFSSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/BFSSuite.scala rename to core/src/test/scala/org/graphframes/lib/BFSSuite.scala diff --git a/src/test/scala/org/graphframes/lib/ConnectedComponentsSuite.scala b/core/src/test/scala/org/graphframes/lib/ConnectedComponentsSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/ConnectedComponentsSuite.scala rename to core/src/test/scala/org/graphframes/lib/ConnectedComponentsSuite.scala diff --git a/src/test/scala/org/graphframes/lib/LabelPropagationSuite.scala b/core/src/test/scala/org/graphframes/lib/LabelPropagationSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/LabelPropagationSuite.scala rename to core/src/test/scala/org/graphframes/lib/LabelPropagationSuite.scala diff --git a/src/test/scala/org/graphframes/lib/PageRankSuite.scala b/core/src/test/scala/org/graphframes/lib/PageRankSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/PageRankSuite.scala rename to core/src/test/scala/org/graphframes/lib/PageRankSuite.scala diff --git a/src/test/scala/org/graphframes/lib/ParallelPersonalizedPageRankSuite.scala b/core/src/test/scala/org/graphframes/lib/ParallelPersonalizedPageRankSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/ParallelPersonalizedPageRankSuite.scala rename to core/src/test/scala/org/graphframes/lib/ParallelPersonalizedPageRankSuite.scala diff --git a/src/test/scala/org/graphframes/lib/PregelSuite.scala b/core/src/test/scala/org/graphframes/lib/PregelSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/PregelSuite.scala rename to core/src/test/scala/org/graphframes/lib/PregelSuite.scala diff --git a/src/test/scala/org/graphframes/lib/SVDPlusPlusSuite.scala b/core/src/test/scala/org/graphframes/lib/SVDPlusPlusSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/SVDPlusPlusSuite.scala rename to core/src/test/scala/org/graphframes/lib/SVDPlusPlusSuite.scala diff --git a/src/test/scala/org/graphframes/lib/ShortestPathsSuite.scala b/core/src/test/scala/org/graphframes/lib/ShortestPathsSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/ShortestPathsSuite.scala rename to core/src/test/scala/org/graphframes/lib/ShortestPathsSuite.scala diff --git a/src/test/scala/org/graphframes/lib/StronglyConnectedComponentsSuite.scala b/core/src/test/scala/org/graphframes/lib/StronglyConnectedComponentsSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/StronglyConnectedComponentsSuite.scala rename to core/src/test/scala/org/graphframes/lib/StronglyConnectedComponentsSuite.scala diff --git a/src/test/scala/org/graphframes/lib/TriangleCountSuite.scala b/core/src/test/scala/org/graphframes/lib/TriangleCountSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/lib/TriangleCountSuite.scala rename to core/src/test/scala/org/graphframes/lib/TriangleCountSuite.scala diff --git a/src/test/scala/org/graphframes/pattern/PatternSuite.scala b/core/src/test/scala/org/graphframes/pattern/PatternSuite.scala similarity index 100% rename from src/test/scala/org/graphframes/pattern/PatternSuite.scala rename to core/src/test/scala/org/graphframes/pattern/PatternSuite.scala diff --git a/python/dev/build_jar.py b/python/dev/build_jar.py index 3bcce30ca..ff081ffd7 100644 --- a/python/dev/build_jar.py +++ b/python/dev/build_jar.py @@ -15,10 +15,7 @@ def build(spark_versions: Sequence[str] = ["3.5.5"]): sbt_build_command = [ sbt_executable, f"-Dspark.version={spark_version}", - "clean", - "package", - "connect/clean", - "connect/package" + "package" ] sbt_build = subprocess.Popen( sbt_build_command, diff --git a/python/tests/conftest.py b/python/tests/conftest.py index beca02b4c..1fee7c475 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -30,8 +30,8 @@ def get_gf_jar_locations() -> Tuple[str, str]: this function will raise an exception! """ project_root = pathlib.Path(__file__).parent.parent.parent - core_dir = project_root / "target" / f"scala-{scala_version}" - connect_dir = project_root / "graphframes-connect" / "target" / f"scala-{scala_version}" + core_dir = project_root / "core" / "target" / f"scala-{scala_version}" + connect_dir = project_root / "connect" / "target" / f"scala-{scala_version}" core_jar: Optional[str] = None connect_jar: Optional[str] = None