From 73cb3fc40ee282e672534b637ff6ab1d47b584f9 Mon Sep 17 00:00:00 2001 From: Jacob Klegar Date: Fri, 8 Jan 2021 20:07:17 -0500 Subject: [PATCH 1/2] add azure libraries Signed-off-by: Jacob Klegar --- .github/workflows/build.yaml | 3 +-- aws_spark/Dockerfile => Dockerfile | 8 +++++--- README.md | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) rename aws_spark/Dockerfile => Dockerfile (65%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 38ed1d1..172c418 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,8 +29,7 @@ jobs: run: bash build.sh - name: Build Docker run: | - cd aws_spark - docker build -t gcr.io/kf-feast/spark-py:v3.0.1 --build-arg BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 . + docker build -t gcr.io/kf-feast/spark-py:v3.0.1 . - name: Push Docker run: | gcloud auth configure-docker diff --git a/aws_spark/Dockerfile b/Dockerfile similarity index 65% rename from aws_spark/Dockerfile rename to Dockerfile index 415603e..2699bdd 100644 --- a/aws_spark/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=seedjeffwan/spark:v2.4.5 +ARG BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 FROM ${BASE_IMAGE} @@ -15,7 +15,9 @@ ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_ ADD https://repo1.maven.org/maven2/com/linkedin/sparktfrecord/spark-tfrecord_2.12/${TFRECORD_VERSION}/spark-tfrecord_2.12-${TFRECORD_VERSION}.jar /opt/spark/jars ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-${GCS_CONNECTOR_VERSION}.jar /opt/spark/jars ADD https://repo1.maven.org/maven2/com/google/cloud/spark/spark-bigquery-with-dependencies_2.12/${BQ_CONNECTOR_VERSION}/spark-bigquery-with-dependencies_2.12-${BQ_CONNECTOR_VERSION}.jar /opt/spark/jars - - +ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.0/hadoop-azure-3.3.0.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/8.6.5/azure-storage-8.6.5.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util-ajax/9.4.26.v20200117/jetty-util-ajax-9.4.26.v20200117.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.26.v20200117/jetty-util-9.4.26.v20200117.jar /opt/spark/jars ENTRYPOINT [ "/opt/entrypoint.sh" ] diff --git a/README.md b/README.md index 9f01716..5f1b8f4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ -This builds a container image for Spark 3.0.1 with two tweaks: +This builds a container image for Spark 3.0.1 with several tweaks: 1. a patch that makes it backwards compatible with spark k8s operator `v1beta2-1.1.2-2.4.5` 2. added AWS libraries +3. added TFRecord support +4. added Azure blob storage libraries + +To build from scratch, first run `build.sh` to build the base image then build the Dockerfile in this repository. The image is publicly available as `gcr.io/kf-feast/spark-py:v3.0.1` From c8fb8a00fa0122739d59e3ecf6dd0bd16936fd6d Mon Sep 17 00:00:00 2001 From: Jacob Klegar Date: Mon, 1 Feb 2021 20:36:25 -0500 Subject: [PATCH 2/2] update README Signed-off-by: Jacob Klegar --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f1b8f4..03f6987 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ This builds a container image for Spark 3.0.1 with several tweaks: 1. a patch that makes it backwards compatible with spark k8s operator `v1beta2-1.1.2-2.4.5` 2. added AWS libraries 3. added TFRecord support -4. added Azure blob storage libraries +4. added GCS libraries +5. added Azure blob storage libraries To build from scratch, first run `build.sh` to build the base image then build the Dockerfile in this repository.