feast-dev · oavdeev · Jan 8, 2021 · Jan 6, 2021 · Jan 6, 2021 · Jan 6, 2021
diff --git a/.dockerignore b/.dockerignore
@@ -2,3 +2,4 @@ docs
 !docs/coverage
 charts
 env
+**/.terraform
diff --git a/.gitignore b/.gitignore
@@ -184,3 +184,8 @@ sdk/python/docs/html
 *_pb2.py
 *_pb2.pyi
 *_pb2_grpc.py
+
+# VSCode
+.bloop
+.metals
+*.code-workspace
diff --git a/Makefile b/Makefile
@@ -156,6 +156,9 @@ build-ci-docker:
 build-jupyter-docker:
 	docker build -t $(REGISTRY)/feast-jupyter:$(VERSION) -f infra/docker/jupyter/Dockerfile .
 
+build-local-test-docker:
+	docker build -t feast:local -f infra/docker/tests/Dockerfile .
+
 # Documentation
 
 install-dependencies-proto-docs:

@@ -0,0 +1,24 @@
+ARG BASE_IMAGE=gcr.io/kf-feast/feast-ci:latest
+
+FROM ${BASE_IMAGE}
+
+RUN mkdir -p /src/sdk /src/spark/ingestion
+
+COPY sdk/python /src/sdk/python
+
+COPY README.md /src/README.md
+
+WORKDIR /src
+
+RUN pip install -r sdk/python/requirements-ci.txt
+
+RUN git init .
+RUN pip install -e sdk/python -U
+RUN pip install "s3fs" "boto3" "urllib3>=1.25.4"
+
+COPY tests /src/tests
+
+RUN pip install -r tests/requirements.txt
+
+COPY infra/scripts /src/infra/scripts
+COPY spark/ingestion/target /src/spark/ingestion/target
@@ -12,7 +12,7 @@ GIT_REMOTE_URL=https://github.com/feast-dev/feast.git
 echo "########## Starting e2e tests for ${GIT_REMOTE_URL} ${GIT_TAG} ###########"
 
 # Note requires running in root feast directory
-source infra/scripts/runner-helper.sh
+source infra/scripts/k8s-common-functions.sh
 
 # Workaround for COPY command in core docker image that pulls local maven repo into the image
 # itself.
@@ -34,8 +34,12 @@ RELEASE=sparkop
 # Delete old helm release and PVCs
 k8s_cleanup "$RELEASE" "$NAMESPACE"
 
+wait_for_images "${DOCKER_REPOSITORY}" "${GIT_TAG}"
+
 # Helm install everything in a namespace
-helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" --namespace "$NAMESPACE"
+helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \
+        --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
+        --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}"
 
 # Delete old test running pod if it exists
 kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true

@@ -1,4 +1,13 @@
-#!/usr/bin/env bash
+#!/bin/bash
+
+set -euo pipefail
+
+function wait_for_images {
+    local DOCKER_REPOSITORY=$1
+    local GIT_TAG=$2
+    # Wait for images to be available in the docker repository; ci is the last image built
+    timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
+}
 
 function k8s_cleanup {
     local RELEASE=$1
@@ -17,8 +26,12 @@ function k8s_cleanup {
 
     # Set a new postgres password. Note that the postgres instance is not available outside
     # the k8s cluster anyway so it doesn't have to be super secure.
-    echo "${STEP_BREADCRUMB} Setting PG password"
-    PG_PASSWORD=$(head -c 59 /dev/urandom | md5sum | head -c 16)
+    echo "${STEP_BREADCRUMB:-} Setting PG password"
+
+    # use either shasum or md5sum, whichever exists
+    SUM=$(which md5sum shasum | grep -v "not found" | tail -n1 || true )
+
+    PG_PASSWORD=$(head -c 59 /dev/urandom | $SUM | head -c 16)
     kubectl delete secret feast-postgresql -n "$NAMESPACE" || true
     kubectl create secret generic feast-postgresql --from-literal=postgresql-password="$PG_PASSWORD" -n "$NAMESPACE"
 }
@@ -30,36 +43,34 @@ function helm_install {
     #   $RELEASE is helm release name
     #   $DOCKER_REPOSITORY is the docker repo containing feast images tagged with $GIT_TAG
     #   ... you can pass additional args to this function that are passed on to helm install
+    #   $NAMESPACE is the namespace name
 
     local RELEASE=$1
     local DOCKER_REPOSITORY=$2
     local GIT_TAG=$3
+    local NAMESPACE=$4
 
-    shift 3
-
-    # Wait for images to be available in the docker repository; ci is the last image built
-    timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
+    shift 4
 
     # We skip statsd exporter and other metrics stuff since we're not using it anyway, and it
     # has some issues with unbound PVCs (that cause kubectl delete pvc to hang).
-    echo "${STEP_BREADCRUMB} Helm installing feast"
+    echo "${STEP_BREADCRUMB:-} Helm installing feast"
 
-    if ! time helm install --wait "$RELEASE" infra/charts/feast \
+    if ! time helm install --wait "$RELEASE" ./infra/charts/feast \
         --timeout 15m \
         --set "feast-jupyter.image.repository=${DOCKER_REPOSITORY}/feast-jupyter" \
         --set "feast-jupyter.image.tag=${GIT_TAG}" \
         --set "feast-online-serving.image.repository=${DOCKER_REPOSITORY}/feast-serving" \
         --set "feast-online-serving.image.tag=${GIT_TAG}" \
         --set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
         --set "feast-jobservice.image.tag=${GIT_TAG}" \
-        --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
-        --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" \
         --set "feast-core.image.repository=${DOCKER_REPOSITORY}/feast-core" \
         --set "feast-core.image.tag=${GIT_TAG}" \
         --set "prometheus-statsd-exporter.enabled=false" \
         --set "prometheus.enabled=false" \
         --set "grafana.enabled=false" \
         --set "feast-jobservice.enabled=false" \
+        --namespace "$NAMESPACE" \
         "$@" ; then
 
         echo "Error during helm install. "
@@ -105,4 +116,4 @@ subjects:
   - kind: ServiceAccount
     name: default
 EOF
-}
+}
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -euo pipefail
+
+NAMESPACE=sparkop
+JOB_NAME=test-runner
+
+# Delete all sparkapplication resources that may be left over from the previous test runs.
+kubectl delete sparkapplication --all -n sparkop || true
+
+JOB_SPEC=$(dirname $0)/test_job.yaml
+
+# Delete previous instance of the job if it exists
+kubectl delete -n ${NAMESPACE} "job/$JOB_NAME"  2>/dev/null || true
+
+# Create the job
+kubectl apply -n ${NAMESPACE} -f "$JOB_SPEC"
+
+# Wait for job to have a pod.
+for i in {1..10}
+do
+	POD=$(kubectl get pods -n ${NAMESPACE} --selector=job-name=$JOB_NAME --output=jsonpath='{.items[0].metadata.name}')
+	if [ ! -z "$POD" ]; then
+		break
+	else
+		sleep 1
+	fi
+done
+
+echo "Waiting for pod to be ready:"
+kubectl wait -n ${NAMESPACE} --for=condition=ContainersReady "pod/$POD" --timeout=60s || true
+
+echo "Job output:"
+kubectl logs -n ${NAMESPACE} -f "job/$JOB_NAME"
+
+# Can't wait for both conditions at once, so wait for complete first then wait for failure
+kubectl wait -n ${NAMESPACE} --for=condition=complete "job/$JOB_NAME" --timeout=60s && exit 0
+kubectl wait -n ${NAMESPACE} --for=condition=failure "job/$JOB_NAME" --timeout=60s && exit 1
@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+STEP_BREADCRUMB='~~~~~~~~'
+
+pushd "$(dirname $0)"
+source k8s-common-functions.sh
+
+# spark k8s test - runs in sparkop namespace (so it doesn't interfere with a concurrently
+# running EMR test).
+NAMESPACE=sparkop
+RELEASE=sparkop
+
+# Clean up old release
+k8s_cleanup "$RELEASE" "$NAMESPACE"
+
+# Helm install everything in a namespace
+helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" --create-namespace
+
+# Delete all sparkapplication resources that may be left over from the previous test runs.
+kubectl delete sparkapplication --all -n "$NAMESPACE" || true
+
+# Make sure the test pod has permissions to create sparkapplication resources
+setup_sparkop_role
+
+echo "DONE"
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export DISABLE_FEAST_SERVICE_FIXTURES=1
+export DISABLE_SERVICE_FIXTURES=1
+
+export FEAST_SPARK_K8S_NAMESPACE=sparkop
+export FEAST_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
+
+# Used by tests
+export AWS_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
+
+cat << SPARK_CONF_END >/tmp/spark_conf.yml
+apiVersion: "sparkoperator.k8s.io/v1beta2"
+kind: SparkApplication
+metadata:
+  namespace: default
+spec:
+  type: Scala
+  mode: cluster
+  image: "gcr.io/kf-feast/spark-py:v3.0.1"
+  imagePullPolicy: Always
+  sparkVersion: "3.0.1"
+  timeToLiveSeconds: 3600
+  pythonVersion: "3"
+  sparkConf:
+    "spark.hadoop.fs.s3a.endpoint": http://minio.minio.svc.cluster.local:9000
+    "spark.hadoop.fs.s3a.path.style.access": "true"
+    "spark.hadoop.fs.s3a.access.key": ${AWS_ACCESS_KEY_ID}
+    "spark.hadoop.fs.s3a.secret.key": ${AWS_SECRET_ACCESS_KEY}
+  restartPolicy:
+    type: Never
+  volumes:
+    - name: "test-volume"
+      hostPath:
+        path: "/tmp"
+        type: Directory
+  driver:
+    cores: 1
+    coreLimit: "1200m"
+    memory: "512m"
+    labels:
+      version: 3.0.1
+    serviceAccount: spark
+    volumeMounts:
+      - name: "test-volume"
+        mountPath: "/tmp"
+  executor:
+    cores: 1
+    instances: 1
+    memory: "512m"
+    labels:
+      version: 3.0.1
+    volumeMounts:
+      - name: "test-volume"
+        mountPath: "/tmp"
+SPARK_CONF_END
+export FEAST_SPARK_K8S_JOB_TEMPLATE_PATH=/tmp/spark_conf.yml
+
+PYTHONPATH=sdk/python pytest tests/e2e/ \
+      --feast-version develop \
+      --core-url sparkop-feast-core:6565 \
+      --serving-url sparkop-feast-online-serving:6566 \
+      --env k8s \
+      --staging-path s3a://feast-staging \
+      --redis-url sparkop-redis-master.sparkop.svc.cluster.local:6379 \
+      --kafka-brokers sparkop-kafka.sparkop.svc.cluster.local:9092 \
+      -m "not bq"
@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-runner
+  namespace: sparkop
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      containers:
+      - name: ubuntu
+        image: feast:local
+        command: ["bash",  "-c", "./infra/scripts/test-end-to-end-local.sh"]
+        imagePullPolicy: Never
+        args:
+        - bash
+        stdin: true
+        stdinOnce: true
+        tty: true
+        env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: minio
+                key: accesskey
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio
+                key: secretkey
+          - name: AWS_DEFAULT_REGION
+            value: us-east-1
+          - name: AWS_S3_SIGNATURE_VERSION
+            value: s3v4
+      restartPolicy: Never
@@ -68,7 +68,16 @@ def _get_optional(option):
 def _k8s_launcher(config: Config) -> JobLauncher:
     from feast.pyspark.launchers import k8s
 
-    return k8s.KubernetesJobLauncher(config=config)
+    staging_location = config.get(opt.SPARK_STAGING_LOCATION)
+    staging_uri = urlparse(staging_location)
+
+    return k8s.KubernetesJobLauncher(
+        namespace=config.get(opt.SPARK_K8S_NAMESPACE),
+        resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None),
+        staging_location=staging_location,
+        incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG),
+        staging_client=get_staging_client(staging_uri.scheme, config),
+    )
 
 
 _launchers = {
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ docs @@
     !docs/coverage
     charts
     env
+    **/.terraform