From e2608e611dd91190f676ff3f819d0d8f3f6261d2 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 08:55:52 -0700 Subject: [PATCH 01/12] gh workflows --- .github/workflows/scala-ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/scala-ci.yml diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml new file mode 100644 index 000000000..bc521f62d --- /dev/null +++ b/.github/workflows/scala-ci.yml @@ -0,0 +1,18 @@ +name: Scala CI +on: + pull_request: + push: +jobs: + test: + runs-on: ubuntu-20.04 + env: + # define Java options for both official sbt and sbt-extras + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + steps: + - uses: actions/checkout@v2 + - uses: olafurpg/setup-scala@v11 + with: + java-version: "zulu@1.8" + - name: Build and Test + run: sbt -v test From 273d55aa2437c253c999537ef3752440713298f4 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 09:18:50 -0700 Subject: [PATCH 02/12] cross --- .github/workflows/scala-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index bc521f62d..dc38a1494 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -1,9 +1,17 @@ name: Scala CI -on: - pull_request: - push: +on: [push, pull_request] jobs: test: + strategy: + fail-fast: false + matrix: + include: + - spark-version: 3.1.2 + scala-version: 2.12.15 + - spark-version: 3.0.3 + scala-version: 2.12.15 + - spark-version: 2.4.8 + scala-version: 2.11.12 runs-on: ubuntu-20.04 env: # define Java options for both official sbt and sbt-extras @@ -15,4 +23,4 @@ jobs: with: java-version: "zulu@1.8" - name: Build and Test - run: sbt -v test + run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} test From 1ce09c0536dd173bffe9bf551eb27d5265e511be Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 10:07:13 -0700 Subject: [PATCH 03/12] python ci --- .github/workflows/python-ci.yml | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/python-ci.yml diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 000000000..f2d7e2cd7 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,34 @@ +name: Python CI +on: [push, pull_request] +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - spark-version: 3.1.2 + - spark-version: 3.0.3 + - spark-version: 2.4.8 + runs-on: ubuntu-20.04 + env: + # define Java options for both official sbt and sbt-extras + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + steps: + - uses: actions/checkout@v2 + - uses: olafurpg/setup-scala@v11 + with: + java-version: "zulu@1.8" + - name: Assembly + run: sbt -v -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Python depencencies + run: | + python -m pip install --upgrade pip + pip install -r ./python/requirements.txt + pip install pyspark==${{ matrix.spark-version }} + - name: Test + run: | + ./python/run-tests.sh From 4848bfd91038c8bdc64c2af63da08ed8c0d9b53d Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 10:19:28 -0700 Subject: [PATCH 04/12] use ivy2 cache --- .github/workflows/python-ci.yml | 5 +++++ .github/workflows/scala-ci.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index f2d7e2cd7..baeecd658 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -19,6 +19,11 @@ jobs: - uses: olafurpg/setup-scala@v11 with: java-version: "zulu@1.8" + - uses: actions/cache@v2 + with: + path: | + ~/.ivy2/cache + key: sbt-ivy-cache-spark-${{ matrix-version}}-scala-${{ matrix.scala-version }} - name: Assembly run: sbt -v -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly - uses: actions/setup-python@v2 diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index dc38a1494..0428ca78a 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -22,5 +22,10 @@ jobs: - uses: olafurpg/setup-scala@v11 with: java-version: "zulu@1.8" + - uses: actions/cache@v2 + with: + path: | + ~/.ivy2/cache + key: sbt-ivy-cache-spark-${{ matrix-version}}-scala-${{ matrix.scala-version }} - name: Build and Test run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} test From 0c6988b676f7bf37783123e5b956436a3b2c2dbb Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 10:23:55 -0700 Subject: [PATCH 05/12] add SPARK_HOME --- .github/workflows/python-ci.yml | 3 ++- .github/workflows/scala-ci.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index baeecd658..ad4089e33 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -23,7 +23,7 @@ jobs: with: path: | ~/.ivy2/cache - key: sbt-ivy-cache-spark-${{ matrix-version}}-scala-${{ matrix.scala-version }} + key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} - name: Assembly run: sbt -v -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly - uses: actions/setup-python@v2 @@ -36,4 +36,5 @@ jobs: pip install pyspark==${{ matrix.spark-version }} - name: Test run: | + export SPARK_HOME=$(python -c "import os, pyspark; print(os.path.dirname(pyspark.__file__))") ./python/run-tests.sh diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 0428ca78a..13bdd24d0 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -26,6 +26,6 @@ jobs: with: path: | ~/.ivy2/cache - key: sbt-ivy-cache-spark-${{ matrix-version}}-scala-${{ matrix.scala-version }} + key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} - name: Build and Test run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} test From 01c099099bb35b2aed70b2ba8cefbca701c6f3e6 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 10:57:38 -0700 Subject: [PATCH 06/12] set envs --- .github/workflows/python-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index ad4089e33..321e14f55 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -7,13 +7,18 @@ jobs: matrix: include: - spark-version: 3.1.2 + scala-version: 2.12.15 - spark-version: 3.0.3 + scala-version: 2.12.15 - spark-version: 2.4.8 + scala-version: 2.11.12 runs-on: ubuntu-20.04 env: # define Java options for both official sbt and sbt-extras JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + SPARK_VERSION: ${{ matrix.spark-version }} + SCALA_VERSION: ${{ matrix.scala-version }} steps: - uses: actions/checkout@v2 - uses: olafurpg/setup-scala@v11 @@ -25,7 +30,7 @@ jobs: ~/.ivy2/cache key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} - name: Assembly - run: sbt -v -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly + run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly - uses: actions/setup-python@v2 with: python-version: 3.8 From 22ddbf00a801d63bbea7c7eaa566d5438a94eb61 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 11:01:56 -0700 Subject: [PATCH 07/12] wheel --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 321e14f55..cac969667 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -36,7 +36,7 @@ jobs: python-version: 3.8 - name: Install Python depencencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel pip install -r ./python/requirements.txt pip install pyspark==${{ matrix.spark-version }} - name: Test From b60091dc177b46d7630acf106a0743055e492750 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 11:05:37 -0700 Subject: [PATCH 08/12] cov --- .github/workflows/scala-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 13bdd24d0..99931d691 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -28,4 +28,5 @@ jobs: ~/.ivy2/cache key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} - name: Build and Test - run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} test + run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} coverage test coverageReport + - uses: codecov/codecov-action@v2 From de48f4b5af16f7ce2aaa015b7bcd7f596e1d1226 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 11:15:33 -0700 Subject: [PATCH 09/12] downgrade scala so coverage works --- .github/workflows/python-ci.yml | 4 ++-- .github/workflows/scala-ci.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index cac969667..a43115116 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -7,9 +7,9 @@ jobs: matrix: include: - spark-version: 3.1.2 - scala-version: 2.12.15 + scala-version: 2.12.12 - spark-version: 3.0.3 - scala-version: 2.12.15 + scala-version: 2.12.12 - spark-version: 2.4.8 scala-version: 2.11.12 runs-on: ubuntu-20.04 diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml index 99931d691..6cc9f8657 100644 --- a/.github/workflows/scala-ci.yml +++ b/.github/workflows/scala-ci.yml @@ -7,9 +7,9 @@ jobs: matrix: include: - spark-version: 3.1.2 - scala-version: 2.12.15 + scala-version: 2.12.12 - spark-version: 3.0.3 - scala-version: 2.12.15 + scala-version: 2.12.12 - spark-version: 2.4.8 scala-version: 2.11.12 runs-on: ubuntu-20.04 From 3e4e6f9a4e7e2b8ce008e3c1a2983e130878eadd Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 11:20:23 -0700 Subject: [PATCH 10/12] remove travis --- .travis.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1b830cfd6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -dist: bionic - -language: scala - -jdk: - - openjdk8 - -cache: - directories: - - $HOME/.ivy2 - - $HOME/.sbt/launchers/ - - $HOME/.cache/spark-versions - -matrix: - include: - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.11.12 SPARK_VERSION=2.4.7 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.12.11 SPARK_VERSION=3.0.2 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.12.11 SPARK_VERSION=3.1.1 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - -before_install: - - ./bin/download_travis_dependencies.sh - -install: - - pyenv global 3.7 - - pip install -r ./python/requirements.txt - -script: - - sbt ++$SCALA_VERSION -Dspark.version=$SPARK_VERSION "set test in assembly := {}" assembly - - sbt ++$SCALA_VERSION -Dspark.version=$SPARK_VERSION coverage test coverageReport - - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh - -after_success: - - bash <(curl -s https://codecov.io/bash) From 829b4a162db5eec33011e2f6ebcda18a4103d08e Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 12:20:33 -0700 Subject: [PATCH 11/12] fix SPARK_HOME --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index a43115116..e5fa8e961 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -41,5 +41,5 @@ jobs: pip install pyspark==${{ matrix.spark-version }} - name: Test run: | - export SPARK_HOME=$(python -c "import os, pyspark; print(os.path.dirname(pyspark.__file__))") + export SPARK_HOME=$(python -c "import os; print(os.path.join(os.path.dirname(os.__file__), 'site-packages', 'pyspark'))") ./python/run-tests.sh From 435536d5fc0cd5a516f3eb7fa620b902eaa93a7f Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Thu, 14 Oct 2021 12:25:44 -0700 Subject: [PATCH 12/12] py37 for spark 2.4 --- .github/workflows/python-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index e5fa8e961..357fbc616 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -8,10 +8,13 @@ jobs: include: - spark-version: 3.1.2 scala-version: 2.12.12 + python-version: 3.8 - spark-version: 3.0.3 scala-version: 2.12.12 + python-version: 3.8 - spark-version: 2.4.8 scala-version: 2.11.12 + python-version: 3.7 runs-on: ubuntu-20.04 env: # define Java options for both official sbt and sbt-extras @@ -33,7 +36,7 @@ jobs: run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Install Python depencencies run: | python -m pip install --upgrade pip wheel