diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 000000000..357fbc616 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,48 @@ +name: Python CI +on: [push, pull_request] +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - spark-version: 3.1.2 + scala-version: 2.12.12 + python-version: 3.8 + - spark-version: 3.0.3 + scala-version: 2.12.12 + python-version: 3.8 + - spark-version: 2.4.8 + scala-version: 2.11.12 + python-version: 3.7 + runs-on: ubuntu-20.04 + env: + # define Java options for both official sbt and sbt-extras + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + SPARK_VERSION: ${{ matrix.spark-version }} + SCALA_VERSION: ${{ matrix.scala-version }} + steps: + - uses: actions/checkout@v2 + - uses: olafurpg/setup-scala@v11 + with: + java-version: "zulu@1.8" + - uses: actions/cache@v2 + with: + path: | + ~/.ivy2/cache + key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} + - name: Assembly + run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python depencencies + run: | + python -m pip install --upgrade pip wheel + pip install -r ./python/requirements.txt + pip install pyspark==${{ matrix.spark-version }} + - name: Test + run: | + export SPARK_HOME=$(python -c "import os; print(os.path.join(os.path.dirname(os.__file__), 'site-packages', 'pyspark'))") + ./python/run-tests.sh diff --git a/.github/workflows/scala-ci.yml b/.github/workflows/scala-ci.yml new file mode 100644 index 000000000..6cc9f8657 --- /dev/null +++ b/.github/workflows/scala-ci.yml @@ -0,0 +1,32 @@ +name: Scala CI +on: [push, pull_request] +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - spark-version: 3.1.2 + scala-version: 2.12.12 + - spark-version: 3.0.3 + scala-version: 2.12.12 + - spark-version: 2.4.8 + scala-version: 2.11.12 + runs-on: ubuntu-20.04 + env: + # define Java options for both official sbt and sbt-extras + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + steps: + - uses: actions/checkout@v2 + - uses: olafurpg/setup-scala@v11 + with: + java-version: "zulu@1.8" + - uses: actions/cache@v2 + with: + path: | + ~/.ivy2/cache + key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }} + - name: Build and Test + run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} coverage test coverageReport + - uses: codecov/codecov-action@v2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1b830cfd6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -dist: bionic - -language: scala - -jdk: - - openjdk8 - -cache: - directories: - - $HOME/.ivy2 - - $HOME/.sbt/launchers/ - - $HOME/.cache/spark-versions - -matrix: - include: - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.11.12 SPARK_VERSION=2.4.7 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.12.11 SPARK_VERSION=3.0.2 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - - env: PYSPARK_PYTHON=python3 SCALA_VERSION=2.12.11 SPARK_VERSION=3.1.1 SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7" - -before_install: - - ./bin/download_travis_dependencies.sh - -install: - - pyenv global 3.7 - - pip install -r ./python/requirements.txt - -script: - - sbt ++$SCALA_VERSION -Dspark.version=$SPARK_VERSION "set test in assembly := {}" assembly - - sbt ++$SCALA_VERSION -Dspark.version=$SPARK_VERSION coverage test coverageReport - - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh - -after_success: - - bash <(curl -s https://codecov.io/bash)