From e3af1836b2973c381fcd6fa9ecddc8460a188e05 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Sat, 30 Jan 2021 15:50:37 +0800 Subject: [PATCH 1/4] Add GCP dependencies to Spark container image Signed-off-by: Khor Shu Heng --- aws_spark/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aws_spark/Dockerfile b/aws_spark/Dockerfile index d1ffb7e..415603e 100644 --- a/aws_spark/Dockerfile +++ b/aws_spark/Dockerfile @@ -5,11 +5,17 @@ FROM ${BASE_IMAGE} ARG HADOOP_AWS_VERSION=3.2.1 ARG AWS_JAVA_SDK_VERSION=1.11.874 ARG TFRECORD_VERSION=0.3.0 +ARG GCS_CONNECTOR_VERSION=2.0.1 +ARG BQ_CONNECTOR_VERSION=0.18.1 USER root # Add HADOOP_AWS_JAR and AWS_JAVA_SDK ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar /opt/spark/jars ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_SDK_VERSION}/aws-java-sdk-bundle-${AWS_JAVA_SDK_VERSION}.jar /opt/spark/jars ADD https://repo1.maven.org/maven2/com/linkedin/sparktfrecord/spark-tfrecord_2.12/${TFRECORD_VERSION}/spark-tfrecord_2.12-${TFRECORD_VERSION}.jar /opt/spark/jars +ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-${GCS_CONNECTOR_VERSION}.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/com/google/cloud/spark/spark-bigquery-with-dependencies_2.12/${BQ_CONNECTOR_VERSION}/spark-bigquery-with-dependencies_2.12-${BQ_CONNECTOR_VERSION}.jar /opt/spark/jars + + ENTRYPOINT [ "/opt/entrypoint.sh" ] From 73cb3fc40ee282e672534b637ff6ab1d47b584f9 Mon Sep 17 00:00:00 2001 From: Jacob Klegar Date: Fri, 8 Jan 2021 20:07:17 -0500 Subject: [PATCH 2/4] add azure libraries Signed-off-by: Jacob Klegar --- .github/workflows/build.yaml | 3 +-- aws_spark/Dockerfile => Dockerfile | 8 +++++--- README.md | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) rename aws_spark/Dockerfile => Dockerfile (65%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 38ed1d1..172c418 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,8 +29,7 @@ jobs: run: bash build.sh - name: Build Docker run: | - cd aws_spark - docker build -t gcr.io/kf-feast/spark-py:v3.0.1 --build-arg BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 . + docker build -t gcr.io/kf-feast/spark-py:v3.0.1 . - name: Push Docker run: | gcloud auth configure-docker diff --git a/aws_spark/Dockerfile b/Dockerfile similarity index 65% rename from aws_spark/Dockerfile rename to Dockerfile index 415603e..2699bdd 100644 --- a/aws_spark/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=seedjeffwan/spark:v2.4.5 +ARG BASE_IMAGE=gcr.io/kf-feast/spark-py-base/spark-py:v3.0.1 FROM ${BASE_IMAGE} @@ -15,7 +15,9 @@ ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_ ADD https://repo1.maven.org/maven2/com/linkedin/sparktfrecord/spark-tfrecord_2.12/${TFRECORD_VERSION}/spark-tfrecord_2.12-${TFRECORD_VERSION}.jar /opt/spark/jars ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-${GCS_CONNECTOR_VERSION}.jar /opt/spark/jars ADD https://repo1.maven.org/maven2/com/google/cloud/spark/spark-bigquery-with-dependencies_2.12/${BQ_CONNECTOR_VERSION}/spark-bigquery-with-dependencies_2.12-${BQ_CONNECTOR_VERSION}.jar /opt/spark/jars - - +ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.0/hadoop-azure-3.3.0.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/8.6.5/azure-storage-8.6.5.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util-ajax/9.4.26.v20200117/jetty-util-ajax-9.4.26.v20200117.jar /opt/spark/jars +ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.26.v20200117/jetty-util-9.4.26.v20200117.jar /opt/spark/jars ENTRYPOINT [ "/opt/entrypoint.sh" ] diff --git a/README.md b/README.md index 9f01716..5f1b8f4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ -This builds a container image for Spark 3.0.1 with two tweaks: +This builds a container image for Spark 3.0.1 with several tweaks: 1. a patch that makes it backwards compatible with spark k8s operator `v1beta2-1.1.2-2.4.5` 2. added AWS libraries +3. added TFRecord support +4. added Azure blob storage libraries + +To build from scratch, first run `build.sh` to build the base image then build the Dockerfile in this repository. The image is publicly available as `gcr.io/kf-feast/spark-py:v3.0.1` From c8fb8a00fa0122739d59e3ecf6dd0bd16936fd6d Mon Sep 17 00:00:00 2001 From: Jacob Klegar Date: Mon, 1 Feb 2021 20:36:25 -0500 Subject: [PATCH 3/4] update README Signed-off-by: Jacob Klegar --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f1b8f4..03f6987 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ This builds a container image for Spark 3.0.1 with several tweaks: 1. a patch that makes it backwards compatible with spark k8s operator `v1beta2-1.1.2-2.4.5` 2. added AWS libraries 3. added TFRecord support -4. added Azure blob storage libraries +4. added GCS libraries +5. added Azure blob storage libraries To build from scratch, first run `build.sh` to build the base image then build the Dockerfile in this repository. From b4143aa30e7116343390f131709812fdd1052003 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Mon, 4 Apr 2022 15:03:40 +0800 Subject: [PATCH 4/4] Remove python 2 from image Signed-off-by: Khor Shu Heng --- .github/workflows/build.yaml | 6 +++--- build.sh | 6 ++---- python.Dockerfile | 42 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) mode change 100644 => 100755 build.sh create mode 100644 python.Dockerfile diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 172c418..fefe70c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,7 +20,7 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} @@ -29,8 +29,8 @@ jobs: run: bash build.sh - name: Build Docker run: | - docker build -t gcr.io/kf-feast/spark-py:v3.0.1 . + docker build -t gcr.io/kf-feast/spark-py:v3.0.1-py39 . - name: Push Docker run: | gcloud auth configure-docker - docker push gcr.io/kf-feast/spark-py:v3.0.1 + docker push gcr.io/kf-feast/spark-py:v3.0.1-py39 diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 index 72bcd95..b00bf74 --- a/build.sh +++ b/build.sh @@ -2,13 +2,11 @@ set -euxo pipefail -git clone https://github.com/apache/spark.git +git clone --branch v3.0.1 --single-branch --depth 1 https://github.com/apache/spark.git cd spark -git checkout v3.0.1 - patch -p1 <../p01.patch ./build/mvn -Pscala-2.12 -Pkubernetes -DskipTests -Phadoop-3.2 -Dhadoop.version=3.2.1 --no-transfer-progress clean package -./bin/docker-image-tool.sh -r gcr.io/kf-feast/spark-py-base -t v3.0.1 -p resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile build +./bin/docker-image-tool.sh -r gcr.io/kf-feast/spark-py-base -t v3.0.1-py39 -p ../python.Dockerfile build diff --git a/python.Dockerfile b/python.Dockerfile new file mode 100644 index 0000000..443ff91 --- /dev/null +++ b/python.Dockerfile @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG base_img + +FROM $base_img +WORKDIR / + +# Reset to root to run installation tasks +USER 0 + +RUN mkdir ${SPARK_HOME}/python +# TODO: Investigate running both pip and pip3 via virtualenvs +RUN apt-get update && \ + apt install -y python3 python3-pip && \ + pip install --upgrade pip setuptools && \ + # Removed the .cache to save space + rm -r /root/.cache && rm -rf /var/cache/apt/* + +COPY python/pyspark ${SPARK_HOME}/python/pyspark +COPY python/lib ${SPARK_HOME}/python/lib + +WORKDIR /opt/spark/work-dir +ENTRYPOINT [ "/opt/entrypoint.sh" ] + +# Specify the User that the actual main process will run as +ARG spark_uid=185 +USER ${spark_uid}