feast-dev · davidheryanto · May 5, 2019 · Apr 24, 2019 · Apr 24, 2019 · Apr 24, 2019
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,9 @@ scratch*
 ### Local Environment ###
 *local*.env
 
+### Secret ###
+**/service_account.json
+
 ### Gradle ###
 **/.gradle
 !gradle/wrapper/gradle-wrapper.jar

diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,117 @@
+cache:
+  directories:
+  - $HOME/.m2 
+
+matrix:
+  include:
+  - stage: unit test
+    name: test ingestion
+    language: java
+    jdk: openjdk8
+    install: echo "skip mvn install"
+    script: mvn --projects ingestion --batch-mode test
+
+  - stage: unit test
+    name: test core
+    language: java
+    jdk: openjdk8
+    install: echo "skip mvn install"
+    script: mvn --projects core --batch-mode test 
+
+  - stage: unit test
+    name: test serving
+    language: java
+    jdk: openjdk8
+    install: echo "skip mvn install"
+    script: mvn --projects serving --batch-mode test
+
+  - stage: unit test
+    name: test cli
+    language: go
+    go: 1.12.x
+    env: GO111MODULE=on
+    script: go test ./cli/feast/...
+
+  - stage: build
+    name: build core
+    language: java
+    jdk: openjdk8
+    services: docker
+    before_install: .travis/decrypt_secrets.sh
+    install: . .travis/install_google_cloud_sdk.sh
+    before_script: .travis/prepare_maven_cache_for_docker.sh
+    script: 
+    - docker build --tag=us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/core/Dockerfile .
+    - docker push us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT}
+    if: type != pull_request
+
+  - stage: build
+    name: build serving
+    language: java
+    jdk: openjdk8
+    services: docker
+    before_install: .travis/decrypt_secrets.sh
+    install: . .travis/install_google_cloud_sdk.sh
+    before_script: .travis/prepare_maven_cache_for_docker.sh
+    script: 
+    - docker build --tag=us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/serving/Dockerfile . 
+    - docker push us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT}
+    if: type != pull_request
+
+  - stage: build
+    name: build cli
+    language: go
+    go: 1.12.x
+    env: 
+    - GO111MODULE=on
+    - FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast
+    before_install: .travis/decrypt_secrets.sh
+    install: . .travis/install_google_cloud_sdk.sh
+    script: 
+    - go build -o ./cli/build/feast ./cli/feast
+    - gsutil cp ./cli/build/feast ${FEAST_CLI_GCS_URI}
+    if: type != pull_request
+
+  - stage: integration test
+    name: test batch and streaming import job
+    language: python
+    python: 3.6
+    env:
+    - BATCH_IMPORT_DATA_LOCAL_PATH=${TRAVIS_BUILD_DIR}/integration-tests/testdata/feature_values/ingestion_1.csv
+    - BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/integration-tests/testdata/feature_values/ingestion_1.csv
+    - FEAST_IMAGE_TAG=${TRAVIS_COMMIT}
+    - FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast
+    - FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER}
+    - FEAST_CORE_URI=localhost:6565
+    - FEAST_SERVING_URI=localhost:6566
+    - KAFKA_BROKERS=localhost:9092
+    - KAFKA_TOPICS=feast-topic
+    before_install: .travis/decrypt_secrets.sh
+    install: 
+    - . .travis/install_google_cloud_sdk.sh
+    - . .travis/install_feast_sdk.sh
+    before_script: 
+    - .travis/start_local_feast.sh
+    - .travis/prepare_testdata.sh
+    script: 
+    - .travis/run_batch_import_and_validate.sh
+    - .travis/run_streaming_import_and_validate.sh
+    after_script: .travis/cleanup_testdata.sh
+    if: type != pull_request
+
+  - stage: deployment test
+    name: test helm deployment 
+    language: minimal
+    env:
+    - BUILD_NUMBER=${TRAVIS_BUILD_NUMBER}
+    - FEAST_IMAGE_TAG=${TRAVIS_COMMIT}
+    - FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER}
+    - RELEASE_NAME=feast-build-${TRAVIS_BUILD_NUMBER}
+    before_install: .travis/decrypt_secrets.sh
+    install: 
+    - . .travis/install_google_cloud_sdk.sh
+    - . .travis/install_helm.sh
+    before_script: envsubst < ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml.template > ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml
+    script: helm install --name ${RELEASE_NAME} --wait --timeout 300 ./charts/feast -f ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml
+    after_script: helm delete --purge ${RELEASE_NAME}
+    if: type != pull_request
diff --git a/.travis/cleanup_testdata.sh b/.travis/cleanup_testdata.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+bq --project_id=kf-feast rm -rf --dataset ${FEAST_WAREHOUSE_DATASET}
+gsutil rm -r ${BATCH_IMPORT_DATA_GCS_PATH}
diff --git a/.travis/decrypt_secrets.sh b/.travis/decrypt_secrets.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# Decrypt Google Cloud service account JSON key for BigQuery and GKE access
+openssl aes-256-cbc \
+-K $encrypted_deb1d355a9cd_key \
+-iv $encrypted_deb1d355a9cd_iv \
+-in ${SCRIPT_DIR}/service_account.json.enc \
+-out ${SCRIPT_DIR}/service_account.json \
+-d
diff --git a/.travis/install_feast_sdk.sh b/.travis/install_feast_sdk.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# This assumes Feast CLI has been built and pushed to GCS in the previous build step
+gsutil cp ${FEAST_CLI_GCS_URI} ${TRAVIS_BUILD_DIR}/cli/build/feast
+chmod +x ${TRAVIS_BUILD_DIR}/cli/build/feast
+export PATH=${TRAVIS_BUILD_DIR}/cli/build:$PATH
+feast config set coreURI ${FEAST_CORE_URI}
+
+pip install -qe ${TRAVIS_BUILD_DIR}/sdk/python
+pip install -qr ${TRAVIS_BUILD_DIR}/integration-tests/testutils/requirements.txt
+export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPT_DIR}/service_account.json
diff --git a/.travis/install_google_cloud_sdk.sh b/.travis/install_google_cloud_sdk.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
+
+wget -qO- ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz
+export PATH=$PWD/google-cloud-sdk/bin:$PATH
+gcloud -q components install kubectl
+gcloud config set project kf-feast
+gcloud -q auth activate-service-account --key-file=${SCRIPT_DIR}/service_account.json
+gcloud -q auth configure-docker
+gcloud -q container clusters get-credentials feast-test-cluster --zone us-central1-a --project kf-feast
diff --git a/.travis/install_helm.sh b/.travis/install_helm.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz
+sudo mv linux-amd64/helm /usr/local/bin/helm
diff --git a/.travis/prepare_maven_cache_for_docker.sh b/.travis/prepare_maven_cache_for_docker.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# This works because Dockerfile for core and serving assumes Maven repository 
+# is located at .m2 directory inside Feast repository root directory
+cp -r $HOME/.m2 ${TRAVIS_BUILD_DIR}/.m2
diff --git a/.travis/prepare_testdata.sh b/.travis/prepare_testdata.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+cd ${TRAVIS_BUILD_DIR}/integration-tests/testdata/import_specs
+envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
+envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
+
+bq --project_id=kf-feast mk --dataset $FEAST_WAREHOUSE_DATASET
+gsutil cp ${BATCH_IMPORT_DATA_LOCAL_PATH} ${BATCH_IMPORT_DATA_GCS_PATH}
diff --git a/.travis/run_batch_import_and_validate.sh b/.travis/run_batch_import_and_validate.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+cd ${TRAVIS_BUILD_DIR}/integration-tests
+
+feast apply entity testdata/entity_specs/entity_1.yaml
+feast apply feature testdata/feature_specs/entity_1.feature_*.yaml
+feast jobs run testdata/import_specs/batch_from_gcs.yaml --wait
+
+python -m testutils.validate_feature_values \
+--entity_spec_file=testdata/entity_specs/entity_1.yaml \
+--feature_spec_files=testdata/feature_specs/entity_1*.yaml \
+--expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
+--expected-serving-values-file=testdata/feature_values/serving_1.csv \
+--bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \
+--feast-serving-url=${FEAST_SERVING_URI} \
+--project=kf-feast
diff --git a/.travis/run_streaming_import_and_validate.sh b/.travis/run_streaming_import_and_validate.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+cd ${TRAVIS_BUILD_DIR}/integration-tests
+feast apply entity testdata/entity_specs/entity_2.yaml
+feast apply feature testdata/feature_specs/entity_2.feature_*.yaml
+feast jobs run testdata/import_specs/stream_from_kafka.yaml &
+
+sleep 20
+
+python -m testutils.kafka_producer \
+--bootstrap_servers=${KAFKA_BROKERS} \
+--topic=${KAFKA_TOPICS} \
+--entity_spec_file=testdata/entity_specs/entity_2.yaml \
+--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
+--feature_values_file=testdata/feature_values/ingestion_2.csv
+
+sleep 20
+
+python -m testutils.validate_feature_values \
+--entity_spec_file=testdata/entity_specs/entity_2.yaml \
+--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
+--expected-serving-values-file=testdata/feature_values/serving_2.csv \
+--feast-serving-url=${FEAST_SERVING_URI}
diff --git a/.travis/service_account.json.enc b/.travis/service_account.json.enc
diff --git a/.travis/start_local_feast.sh b/.travis/start_local_feast.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+COLOR="\033[0;32m$(tput bold)"
+NO_COLOR="\033[0m$(tput sgr0)"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+echo -e "${COLOR}Starting Postgres...${NO_COLOR}"
+docker run --net=host --env=POSTGRES_PASSWORD=password --detach postgres:11.2-alpine
+
+echo -e "${COLOR}Starting Redis...${NO_COLOR}"
+docker run --net=host --detach redis:5.0-alpine
+
+echo -e "${COLOR}Starting Zookeeper...${NO_COLOR}"
+docker run --net=host --env=ZOOKEEPER_CLIENT_PORT=2181 \
+--detach confluentinc/cp-zookeeper:5.2.1
+
+echo -e "${COLOR}Starting Kafka...${NO_COLOR}"
+docker run --net=host \
+--env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \
+--env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
+--env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
+--detach confluentinc/cp-kafka:5.2.1
+
+echo -e "${COLOR}Starting Feast Core...${NO_COLOR}"
+docker run --name core --net host \
+--env=JOB_WORKSPACE=/tmp \
+--env=STORE_WAREHOUSE_TYPE=bigquery \
+--env=STORE_WAREHOUSE_OPTIONS='{"project":"kf-feast","dataset":"'${FEAST_WAREHOUSE_DATASET}'","tempLocation":"gs://feast-templocation-kf-feast"}' \
+--env=STORE_SERVING_TYPE=redis \
+--env=STORE_SERVING_OPTIONS='{"host": "localhost", "port": "6379"}' \
+--env=GOOGLE_APPLICATION_CREDENTIALS=/etc/google_cloud/service_account.json \
+--volume=${SCRIPT_DIR}/service_account.json:/etc/google_cloud/service_account.json \
+--detach us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG}
+sleep 20
+
+echo -e "${COLOR}Starting Feast Serving...${NO_COLOR}"
+docker run --name serving --net host \
+--env=FEAST_SERVING_HTTP_PORT=8081  \
+--detach us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG}
diff --git a/Dockerfiles/README.md b/Dockerfiles/README.md
@@ -0,0 +1 @@
+TODO
diff --git a/Dockerfiles/core/Dockerfile b/Dockerfiles/core/Dockerfile
@@ -0,0 +1,16 @@
+FROM maven:3.6-jdk-8-slim as builder
+ARG REVISION=dev
+COPY . /build
+WORKDIR /build
+ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
+RUN mvn --projects core,ingestion -Drevision=$REVISION -DskipTests=true --batch-mode package
+
+FROM openjdk:8-jre as production
+ARG REVISION=dev
+COPY --from=builder /build/core/target/feast-core-$REVISION.jar /usr/share/feast/feast-core.jar
+COPY --from=builder /build/ingestion/target/feast-ingestion-$REVISION.jar /usr/share/feast/feast-ingestion.jar
+ENV JOB_EXECUTABLE=/usr/share/feast/feast-ingestion.jar
+ENTRYPOINT ["/usr/bin/java", \
+"-XX:+UnlockExperimentalVMOptions", \
+"-XX:+UseCGroupMemoryLimitForHeap", \
+"-jar", "/usr/share/feast/feast-core.jar"]
diff --git a/Dockerfiles/serving/Dockerfile b/Dockerfiles/serving/Dockerfile
@@ -0,0 +1,16 @@
+FROM maven:3.6-jdk-8-slim as builder
+ARG REVISION=dev
+COPY . /build 
+WORKDIR /build
+ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
+RUN mvn --projects serving -Drevision=$REVISION -DskipTests=true --batch-mode package
+
+FROM openjdk:8-jre-alpine as production
+ARG REVISION=dev
+COPY --from=builder /build/serving/target/feast-serving-$REVISION.jar /usr/share/feast/feast-serving.jar
+ENTRYPOINT ["/usr/bin/java", \
+      "-XX:+UseG1GC", \
+      "-XX:+UseStringDeduplication", \
+      "-XX:+UnlockExperimentalVMOptions", \
+      "-XX:+UseCGroupMemoryLimitForHeap", \
+      "-jar", "/usr/share/feast/feast-serving.jar"]
diff --git a/charts/dist/feast-0.1.0.tgz b/charts/dist/feast-0.1.0.tgz
diff --git a/charts/dist/index.yaml b/charts/dist/index.yaml
diff --git a/charts/feast/charts/postgresql-0.19.0.tgz b/charts/feast/charts/postgresql-0.19.0.tgz
diff --git a/charts/feast/charts/postgresql-3.17.0.tgz b/charts/feast/charts/postgresql-3.17.0.tgz
diff --git a/charts/feast/charts/redis-6.0.0.tgz b/charts/feast/charts/redis-6.0.0.tgz
diff --git a/charts/feast/charts/redis-6.4.4.tgz b/charts/feast/charts/redis-6.4.4.tgz
diff --git a/charts/feast/requirements.lock b/charts/feast/requirements.lock
@@ -1,9 +1,9 @@
 dependencies:
 - name: postgresql
   repository: https://kubernetes-charts.storage.googleapis.com
-  version: 0.19.0
+  version: 3.17.0
 - name: redis
   repository: https://kubernetes-charts.storage.googleapis.com
-  version: 6.0.0
-digest: sha256:cb4801425432b544a54462d657daf5ffb8bc18eb5f25b39ef5560a66ae4975eb
-generated: 2019-02-07T10:54:57.715495851+08:00
+  version: 6.4.4
+digest: sha256:e1fb31a7beda17b642a0e1a5369bb8184a7206573e48f07b1bb4ebe35ba9bead
+generated: 2019-04-24T16:00:44.597053887+08:00
diff --git a/charts/feast/requirements.yaml b/charts/feast/requirements.yaml
@@ -1,9 +1,9 @@
 dependencies:
-  - name: postgresql
-    version: ^0.19.0
-    repository: "@stable"
-    condition: postgres.provision
-  - name: redis
-    version: ^6.0.0
-    repository: "@stable"
-    condition: redis.provision
+- name: postgresql
+  version: 3.17.0
+  repository: "@stable"
+  condition: postgres.provision
+- name: redis
+  version: 6.4.4
+  repository: "@stable"
+  condition: redis.provision