Skip to content

Commit 82f7979

Browse files
author
William Tang
authored
Update CI actions and dependencies (#425)
* ci: Update spark versions * ci: Update ubuntu to 22.04 * ci: Update actions * ci: Use openjdk 11 * ci: Use python 3.9 * python: Update nose to 1.3.7
1 parent 1e1dbd3 commit 82f7979

6 files changed

Lines changed: 26 additions & 24 deletions

File tree

‎.github/workflows/python-ci.yml‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,41 @@ jobs:
66
fail-fast: false
77
matrix:
88
include:
9-
- spark-version: 3.3.0
9+
- spark-version: 3.3.2
1010
scala-version: 2.12.12
11-
- spark-version: 3.2.2
11+
python-version: 3.9
12+
- spark-version: 3.2.3
1213
scala-version: 2.12.12
14+
python-version: 3.9
1315
- spark-version: 3.1.3
1416
scala-version: 2.12.12
15-
python-version: 3.8
17+
python-version: 3.9
1618
- spark-version: 3.0.3
1719
scala-version: 2.12.12
18-
python-version: 3.8
19-
runs-on: ubuntu-20.04
20+
python-version: 3.9
21+
runs-on: ubuntu-22.04
2022
env:
2123
# define Java options for both official sbt and sbt-extras
2224
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
2325
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
2426
SPARK_VERSION: ${{ matrix.spark-version }}
2527
SCALA_VERSION: ${{ matrix.scala-version }}
2628
steps:
27-
- uses: actions/checkout@v2
28-
- uses: olafurpg/setup-scala@v11
29+
- uses: actions/checkout@v3
30+
- uses: olafurpg/setup-scala@v13
2931
with:
30-
java-version: "zulu@1.8"
31-
- uses: actions/cache@v2
32+
java-version: "openjdk@1.11"
33+
- uses: actions/cache@v3
3234
with:
3335
path: |
3436
~/.ivy2/cache
3537
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
3638
- name: Assembly
3739
run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly
38-
- uses: actions/setup-python@v2
40+
- uses: actions/setup-python@v4
3941
with:
4042
python-version: ${{ matrix.python-version }}
41-
- name: Install Python depencencies
43+
- name: Install python dependencies
4244
run: |
4345
python -m pip install --upgrade pip wheel
4446
pip install -r ./python/requirements.txt

‎.github/workflows/scala-ci.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,29 @@ jobs:
66
fail-fast: false
77
matrix:
88
include:
9-
- spark-version: 3.3.0
9+
- spark-version: 3.3.2
1010
scala-version: 2.12.12
11-
- spark-version: 3.2.2
11+
- spark-version: 3.2.3
1212
scala-version: 2.12.12
1313
- spark-version: 3.1.3
1414
scala-version: 2.12.12
1515
- spark-version: 3.0.3
1616
scala-version: 2.12.12
17-
runs-on: ubuntu-20.04
17+
runs-on: ubuntu-22.04
1818
env:
1919
# define Java options for both official sbt and sbt-extras
2020
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
2121
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
2222
steps:
23-
- uses: actions/checkout@v2
24-
- uses: olafurpg/setup-scala@v11
23+
- uses: actions/checkout@v3
24+
- uses: olafurpg/setup-scala@v13
2525
with:
26-
java-version: "zulu@1.8"
27-
- uses: actions/cache@v2
26+
java-version: "openjdk@1.11"
27+
- uses: actions/cache@v3
2828
with:
2929
path: |
3030
~/.ivy2/cache
3131
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
3232
- name: Build and Test
3333
run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} coverage test coverageReport
34-
- uses: codecov/codecov-action@v2
34+
- uses: codecov/codecov-action@v3

‎Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ubuntu:20.04
1+
FROM ubuntu:22.04
22

33
ARG PYTHON_VERSION=3.8
44
ARG DEBIAN_FRONTEND=noninteractive
@@ -9,7 +9,7 @@ RUN apt-get update && \
99

1010
# Install Spark and update env variables.
1111
ENV SCALA_VERSION 2.12.15
12-
ENV SPARK_VERSION "3.3.0"
12+
ENV SPARK_VERSION "3.3.2"
1313
ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop3.2"
1414
ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-${SPARK_VERSION}/${SPARK_BUILD}.tgz"
1515
RUN wget --quiet "$SPARK_BUILD_URL" -O /tmp/spark.tgz && \

‎build.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import ReleaseTransformations._
55

66
resolvers += "Spark snapshot repository" at "https://repository.apache.org/snapshots/"
77

8-
val sparkVer = sys.props.getOrElse("spark.version", "3.3.0")
8+
val sparkVer = sys.props.getOrElse("spark.version", "3.3.2")
99
val sparkBranch = sparkVer.substring(0, 3)
1010
val defaultScalaVer = sparkBranch match {
1111
case "3.3" => "2.12.15"

‎dev/release.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def verify(prompt, interactive):
3939
@click.option("--publish-docs", type=bool, default=PUBLISH_DOCS_DEFAULT, show_default=True,
4040
help="Publish docs to github-pages.")
4141
@click.option("--spark-version", multiple=True, show_default=True,
42-
default=["3.0.3", "3.1.3", "3.2.2", "3.3.0"])
42+
default=["3.0.3", "3.1.3", "3.2.3", "3.3.2"])
4343
def main(release_version, next_version, publish_to, no_prompt, git_remote, publish_docs,
4444
spark_version):
4545
interactive = not no_prompt

‎python/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# This file should list any python package dependencies.
2-
nose==1.3.3
2+
nose==1.3.7
33
numpy>=1.7

0 commit comments

Comments
 (0)