diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 38ed1d1..172c418 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,8 +29,7 @@ jobs: run: bash build.sh - name: Build Docker run: | - cd aws_spark - docker build -t gcr.io/kf-feast/spark-py:v3.0.1 --build-arg BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 . + docker build -t gcr.io/kf-feast/spark-py:v3.0.1 . - name: Push Docker run: | gcloud auth configure-docker diff --git a/aws_spark/Dockerfile b/Dockerfile similarity index 65% rename from aws_spark/Dockerfile rename to Dockerfile index 415603e..2699bdd 100644 --- a/aws_spark/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=seedjeffwan/spark:v2.4.5 +ARG BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 FROM ${BASE_IMAGE} @@ -15,7 +15,9 @@ ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_ ADD https://repo1.maven.org/maven2/com/linkedin/sparktfrecord/spark-tfrecord_2.12/${TFRECORD_VERSION}/spark-tfrecord_2.12-${TFRECORD_VERSION}.jar /opt/spark/jars ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-${GCS_CONNECTOR_VERSION}.jar /opt/spark/jars ADD https://repo1.maven.org/maven2/com/google/cloud/spark/spark-bigquery-with-dependencies_2.12/${BQ_CONNECTOR_VERSION}/spark-bigquery-with-dependencies_2.12-${BQ_CONNECTOR_VERSION}.jar /opt/spark/jars - - +ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.0/hadoop-azure-3.3.0.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/8.6.5/azure-storage-8.6.5.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util-ajax/9.4.26.v20200117/jetty-util-ajax-9.4.26.v20200117.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.26.v20200117/jetty-util-9.4.26.v20200117.jar /opt/spark/jars ENTRYPOINT [ "/opt/entrypoint.sh" ] diff --git a/README.md b/README.md index 9f01716..03f6987 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ -This builds a container image for Spark 3.0.1 with two tweaks: +This builds a container image for Spark 3.0.1 with several tweaks: 1. a patch that makes it backwards compatible with spark k8s operator `v1beta2-1.1.2-2.4.5` 2. added AWS libraries +3. added TFRecord support +4. added GCS libraries +5. added Azure blob storage libraries + +To build from scratch, first run `build.sh` to build the base image then build the Dockerfile in this repository. The image is publicly available as `gcr.io/kf-feast/spark-py:v3.0.1`