From c768acf38023de50fda624963730cf18992de834 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 16:56:48 +0800 Subject: [PATCH 01/23] Update prow config and test script so it works for Feast 0.3 --- .prow/config.yaml | 55 ++++++------------- .prow/scripts/run_test.sh | 98 ++++++++++++++++++++++++++++++++++ .prow/scripts/run_unit_test.sh | 63 ---------------------- 3 files changed, 113 insertions(+), 103 deletions(-) create mode 100755 .prow/scripts/run_test.sh delete mode 100755 .prow/scripts/run_unit_test.sh diff --git a/.prow/config.yaml b/.prow/config.yaml index 225e89ee5b1..4a21bece186 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -50,7 +50,7 @@ tide: # presubmits list Prow jobs that run on pull requests presubmits: gojek/feast: - - name: unit-test-core + - name: test-core-and-ingestion decorate: true always_run: true spec: @@ -67,66 +67,41 @@ presubmits: env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "core"] + command: [".prow/scripts/run_test.sh", "--component", "core-ingestion"] - - name: unit-test-ingestion + - name: test-serving decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - image: maven:3.6-jdk-8 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"] + command: [".prow/scripts/run_test.sh", "--component", "serving"] - - name: unit-test-serving + - name: test-java-sdk decorate: true always_run: true spec: containers: - image: maven:3.6-jdk-8 - command: [".prow/scripts/run_unit_test.sh", "--component", "serving"] + command: [".prow/scripts/run_test.sh", "--component", "java-sdk"] - - name: unit-test-cli + - name: test-python-sdk decorate: true always_run: true spec: containers: - - image: golang:1.12 - env: - - name: GO111MODULE - value: "on" - command: [".prow/scripts/run_unit_test.sh", "--component", "cli"] + - image: python:3.6 + command: [".prow/scripts/run_test.sh", "--component", "python-sdk"] - - name: unit-test-python-sdk + - name: test-golang-sdk decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - - image: python:3.6 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "python-sdk"] + - image: golang:1.13 + command: [".prow/scripts/run_test.sh", "--component", "golang-sdk"] - - name: integration-test + - name: test-end-to-end decorate: true always_run: true spec: @@ -175,7 +150,7 @@ presubmits: exit ${TEST_EXIT_CODE} # TODO: do a release when a git tag is pushed -# postsubmits list Prow jobs that run on every push # +# postsubmits list Prow jobs that run on every push # postsubmits: -# gojek/feast: +# gojek/feast: \ No newline at end of file diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh new file mode 100755 index 00000000000..dfed6188307 --- /dev/null +++ b/.prow/scripts/run_test.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +usage() +{ + echo "Run test on a Feast component. + +Usage: run_test.sh --component + + is one of: +- core-ingestion (core depends on ingestion so they are tested together) +- serving +- java-sdk +- python-sdk +- golang-sdk + +This script also runs commands before and after the main test task, such as: +- Download cached Maven packages for faster tests +- Saving the test output report so it can be viewed with Spyglass UI in Prow. + By default, the configured log path is "/logs" and test artifacts should + be saved to "/logs/artifacts" directory. +" +} + +while [ "$1" != "" ]; do + case "$1" in + --component ) COMPONENT="$2"; shift;; + * ) usage; exit 1 + esac + shift +done + +if [[ ! ${COMPONENT} ]]; then + usage; exit 1; +fi + +. .prow/scripts/install_google_cloud_sdk.sh + +if [[ ${COMPONENT} == "core-ingestion" ]]; then + + .prow/scripts/prepare_maven_cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + + # Core depends on Ingestion so they are tested together + mvn --define skipTests=true --projects core,ingestion clean install + mvn --projects core,ingestion test + TEST_EXIT_CODE=$? + + mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports + cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/* + cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/* + +elif [[ ${COMPONENT} == "serving" ]]; then + + .prow/scripts/prepare_maven_cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + + mvn --define skipTests=true --projects serving clean install + mvn --projects serving test + TEST_EXIT_CODE=$? + + cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +elif [[ ${COMPONENT} == "java-sdk" ]]; then + + .prow/scripts/prepare_maven_cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + + # Core depends on Ingestion so they are tested together + mvn --define skipTests=true --projects sdk/java clean install + mvn --projects sdk/java test + TEST_EXIT_CODE=$? + + cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +elif [[ ${COMPONENT} == "python-sdk" ]]; then + + cd sdk/python + pip install -r requirements-test.txt + python -m pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml + TEST_EXIT_CODE=$? + +elif [[ ${COMPONENT} == "golang-sdk" ]]; then + + go test -v 2>&1 | tee /tmp/test_output + TEST_EXIT_CODE=$? + + go get -u github.com/jstemmer/go-junit-report + cat /tmp/test_output | go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml + + +else + usage; exit 1 +fi + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/run_unit_test.sh b/.prow/scripts/run_unit_test.sh deleted file mode 100755 index d2f6ea3255c..00000000000 --- a/.prow/scripts/run_unit_test.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -# This script will run unit test for a specific Feast component: -# - core, ingestion, serving or cli -# -# This script includes the pre and post test scripts, such as -# - downloading maven cache repository -# - saving the test output report so it can be viewed with Spyglass in Prow - -# Bucket in GCS used for running unit tests, when the unit tests need an -# actual running GCS (e.g. because there is no existing mock implementation of the function to test) -TEST_BUCKET=feast-templocation-kf-feast - -usage() -{ - echo "usage: run_unit_test.sh - --component {core, ingestion, serving, cli}" -} - -while [ "$1" != "" ]; do - case "$1" in - --component ) COMPONENT="$2"; shift;; - * ) usage; exit 1 - esac - shift -done - -if [[ ! ${COMPONENT} ]]; then - usage; exit 1; -fi - -. .prow/scripts/install_google_cloud_sdk.sh - -if [[ ${COMPONENT} == "core" ]] || [[ ${COMPONENT} == "ingestion" ]] || [[ ${COMPONENT} == "serving" ]]; then - - .prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ - mvn --projects ${COMPONENT} -Dtestbucket=feast-templocation-kf-feast test - TEST_EXIT_CODE=$? - cp -r ${COMPONENT}/target/surefire-reports /logs/artifacts/surefire-reports - -elif [[ ${COMPONENT} == "cli" ]]; then - - # https://stackoverflow.com/questions/6871859/piping-command-output-to-tee-but-also-save-exit-code-of-command - set -o pipefail - - go get -u github.com/jstemmer/go-junit-report - go test -v ./cli/feast/... 2>&1 | tee test_output - TEST_EXIT_CODE=$? - cat test_output | ${GOPATH}/bin/go-junit-report > ${ARTIFACTS}/unittest-cli-report.xml - -elif [[ ${COMPONENT} == "python-sdk" ]]; then - - cd sdk/python - pip install -r requirements-test.txt - pip install . - pytest ./tests --junitxml=${ARTIFACTS}/unittest-pythonsdk-report.xml - TEST_EXIT_CODE=$? - -else - usage; exit 1 -fi - -exit ${TEST_EXIT_CODE} From 4e747ef8ccaed28ea28413231a1c3f307936d656 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 17:03:47 +0800 Subject: [PATCH 02/23] Update CI test script for golang to use correct path --- .prow/scripts/run_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index dfed6188307..4f488706025 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -84,11 +84,12 @@ elif [[ ${COMPONENT} == "python-sdk" ]]; then elif [[ ${COMPONENT} == "golang-sdk" ]]; then + cd sdk/go go test -v 2>&1 | tee /tmp/test_output TEST_EXIT_CODE=$? go get -u github.com/jstemmer/go-junit-report - cat /tmp/test_output | go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml + cat /tmp/test_output | /go/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml else From eac9d8f415e0226d9a2036fb9fc084cdae2d564d Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 17:08:26 +0800 Subject: [PATCH 03/23] Set pipefail option so test exit code is correct For those commands that pipe the test output --- .prow/scripts/run_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 4f488706025..7f2aa084158 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -o pipefail + # Default artifact location setting in Prow jobs LOGS_ARTIFACT_PATH=/logs/artifacts From f05be1821ee923c48f340cd5781ccd326d1c0e52 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 18:19:16 +0800 Subject: [PATCH 04/23] Update group id and version grpc-spring-boot-starter dependency - Newer version is hosted on Maven central which is more maintained than other repo --- .prow/scripts/run_test.sh | 4 ++-- core/pom.xml | 4 ++-- pom.xml | 4 ++-- serving/pom.xml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 7f2aa084158..dd06e5107d4 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -81,7 +81,8 @@ elif [[ ${COMPONENT} == "python-sdk" ]]; then cd sdk/python pip install -r requirements-test.txt - python -m pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml + pip install . + pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? elif [[ ${COMPONENT} == "golang-sdk" ]]; then @@ -93,7 +94,6 @@ elif [[ ${COMPONENT} == "golang-sdk" ]]; then go get -u github.com/jstemmer/go-junit-report cat /tmp/test_output | /go/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml - else usage; exit 1 fi diff --git a/core/pom.xml b/core/pom.xml index 7a02875266f..a53ee8ec514 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -101,9 +101,9 @@ org.springframework.boot spring-boot-starter-log4j2 - + - org.lognet + io.github.lognet grpc-spring-boot-starter diff --git a/pom.xml b/pom.xml index f3735945d25..4767708a239 100644 --- a/pom.xml +++ b/pom.xml @@ -219,9 +219,9 @@ - org.lognet + io.github.lognet grpc-spring-boot-starter - 2.4.1 + 3.0.2 diff --git a/serving/pom.xml b/serving/pom.xml index 06ba6b225ce..5d9af6bd389 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -99,9 +99,9 @@ true - + - org.lognet + io.github.lognet grpc-spring-boot-starter From 5bece03c6083e50bfcff8dfc531abb91932f61a5 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 18:25:56 +0800 Subject: [PATCH 05/23] Add JVM heap settings for Maven surefire Otherwise it fails to run some tests due to insufficient memory --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4767708a239..c42833b02b6 100644 --- a/pom.xml +++ b/pom.xml @@ -353,7 +353,7 @@ maven-surefire-plugin 2.22.1 - -Djdk.net.URLClassPath.disableClassPathURLCheck=true + -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true IntegrationTest From cf712974e27288458ad936ccff189ccb7c41f1ba Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 18:46:12 +0800 Subject: [PATCH 06/23] Fix potential setup/teardown error when running ImportJobTest - Set project to empty (DataflowOptions in Beam require project to be not null) - Ignore error when shutting down Kafka server during tear down. This should not affect test result. --- ingestion/src/test/java/feast/ingestion/ImportJobTest.java | 1 + ingestion/src/test/java/feast/test/TestUtil.java | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java index 1e9f0bf0129..d08e569518c 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java @@ -116,6 +116,7 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() options.setStoreJson( Collections.singletonList( JsonFormat.printer().omittingInsignificantWhitespace().print(redis))); + options.setProject(""); options.setBlockOnRun(false); int inputSize = 4096; diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java index 8e30edb9aef..8c03d6da172 100644 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ b/ingestion/src/test/java/feast/test/TestUtil.java @@ -92,7 +92,11 @@ public static void start(String kafkaHost, int kafkaPort, short kafkaReplication public static void stop() { if (server != null) { - server.shutdown(); + try { + server.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } } } } From ce6d5546186fba3d6004d6716590bef9d471bf98 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 22:06:47 +0800 Subject: [PATCH 07/23] Update remote URI to download cached Maven packages - This tar archive contains packages used by Feast 0.3 rather than previous version of Feast --- .prow/scripts/run_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index dd06e5107d4..4febab4761e 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -43,7 +43,7 @@ fi if [[ ${COMPONENT} == "core-ingestion" ]]; then .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ # Core depends on Ingestion so they are tested together mvn --define skipTests=true --projects core,ingestion clean install @@ -57,7 +57,7 @@ if [[ ${COMPONENT} == "core-ingestion" ]]; then elif [[ ${COMPONENT} == "serving" ]]; then .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ mvn --define skipTests=true --projects serving clean install mvn --projects serving test @@ -68,7 +68,7 @@ elif [[ ${COMPONENT} == "serving" ]]; then elif [[ ${COMPONENT} == "java-sdk" ]]; then .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ # Core depends on Ingestion so they are tested together mvn --define skipTests=true --projects sdk/java clean install From ca70a9dded3b4083ba5c24e396c46c9b7d4b814b Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 23:00:16 +0800 Subject: [PATCH 08/23] Fix path to logs artifacts --- .prow/scripts/install_google_cloud_sdk.sh | 2 +- .prow/scripts/prepare_maven_cache.sh | 6 ++++++ .prow/scripts/run_test.sh | 14 +++++++------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.prow/scripts/install_google_cloud_sdk.sh b/.prow/scripts/install_google_cloud_sdk.sh index 0865f6085c3..7684760dee9 100755 --- a/.prow/scripts/install_google_cloud_sdk.sh +++ b/.prow/scripts/install_google_cloud_sdk.sh @@ -23,7 +23,7 @@ while [ "$1" != "" ]; do shift done -GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz +GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-266.0.0-linux-x86_64.tar.gz GOOGLE_PROJECT_ID=kf-feast KUBE_CLUSTER_NAME=primary-test-cluster KUBE_CLUSTER_ZONE=us-central1-a diff --git a/.prow/scripts/prepare_maven_cache.sh b/.prow/scripts/prepare_maven_cache.sh index e5113389426..c9704878e0a 100755 --- a/.prow/scripts/prepare_maven_cache.sh +++ b/.prow/scripts/prepare_maven_cache.sh @@ -23,5 +23,11 @@ done if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi +# Install Google Cloud SDK if gsutil command not exists +if [[ ! $(command -v gsutil) ]]; then + CURRENT_DIR=$(dirname "$BASH_SOURCE") + . "${CURRENT_DIR}"/install_google_cloud_sdk.sh +fi + gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar tar xf /tmp/.m2.tar -C ${OUTPUT_DIR} diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 4febab4761e..6dd5eb7af01 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -set -o pipefail - # Default artifact location setting in Prow jobs LOGS_ARTIFACT_PATH=/logs/artifacts +set -o pipefail + usage() { echo "Run test on a Feast component. @@ -38,8 +38,6 @@ if [[ ! ${COMPONENT} ]]; then usage; exit 1; fi -. .prow/scripts/install_google_cloud_sdk.sh - if [[ ${COMPONENT} == "core-ingestion" ]]; then .prow/scripts/prepare_maven_cache.sh \ @@ -51,8 +49,8 @@ if [[ ${COMPONENT} == "core-ingestion" ]]; then TEST_EXIT_CODE=$? mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports - cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/* - cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/* + cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ + cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ elif [[ ${COMPONENT} == "serving" ]]; then @@ -92,7 +90,9 @@ elif [[ ${COMPONENT} == "golang-sdk" ]]; then TEST_EXIT_CODE=$? go get -u github.com/jstemmer/go-junit-report - cat /tmp/test_output | /go/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml + env + ls -lh $GOPATH/bin + cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml else usage; exit 1 From d79dfc0a5b823ccfcfc68b55e7c509be6d45c1b6 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 23:27:05 +0800 Subject: [PATCH 09/23] Update maven enforcer rule for Maven and JDK versions - Allow more ranges of versions as long as they are not breaking changes --- .prow/scripts/run_test.sh | 3 --- pom.xml | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 6dd5eb7af01..49be0c96da0 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -68,7 +68,6 @@ elif [[ ${COMPONENT} == "java-sdk" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - # Core depends on Ingestion so they are tested together mvn --define skipTests=true --projects sdk/java clean install mvn --projects sdk/java test TEST_EXIT_CODE=$? @@ -90,8 +89,6 @@ elif [[ ${COMPONENT} == "golang-sdk" ]]; then TEST_EXIT_CODE=$? go get -u github.com/jstemmer/go-junit-report - env - ls -lh $GOPATH/bin cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml else diff --git a/pom.xml b/pom.xml index 6d61c059d41..c88606df6f5 100644 --- a/pom.xml +++ b/pom.xml @@ -392,10 +392,10 @@ - 3.0.5 + [3.5,4.0) - 1.8.0 + [1.8,1.9) From df5a47d5e891c5b2e943fdcd0ceb67b92be4359e Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 23:30:00 +0800 Subject: [PATCH 10/23] Use batch mode when initializing Maven for cleaner build log --- .prow/scripts/run_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 49be0c96da0..5b8765dd53c 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -44,7 +44,7 @@ if [[ ${COMPONENT} == "core-ingestion" ]]; then --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ # Core depends on Ingestion so they are tested together - mvn --define skipTests=true --projects core,ingestion clean install + mvn --define skipTests=true --projects core,ingestion --batch-mode clean install mvn --projects core,ingestion test TEST_EXIT_CODE=$? @@ -57,7 +57,7 @@ elif [[ ${COMPONENT} == "serving" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - mvn --define skipTests=true --projects serving clean install + mvn --define skipTests=true --projects serving --batch-mode clean install mvn --projects serving test TEST_EXIT_CODE=$? @@ -68,7 +68,7 @@ elif [[ ${COMPONENT} == "java-sdk" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - mvn --define skipTests=true --projects sdk/java clean install + mvn --define skipTests=true --projects sdk/java --batch-mode clean install mvn --projects sdk/java test TEST_EXIT_CODE=$? From dabd984792ffa48d4e271f81fcb21e40617ed6f7 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 24 Oct 2019 23:54:48 +0800 Subject: [PATCH 11/23] Skip maven enforcer when running test for specific project --- .prow/scripts/run_test.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 5b8765dd53c..2c36417d67d 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -44,8 +44,9 @@ if [[ ${COMPONENT} == "core-ingestion" ]]; then --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ # Core depends on Ingestion so they are tested together - mvn --define skipTests=true --projects core,ingestion --batch-mode clean install - mvn --projects core,ingestion test + # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level + mvn --projects core,ingestion --batch-mode --define skipTests=true --define enforcer.skip=true clean install + mvn --projects core,ingestion --define enforcer.skip=true test TEST_EXIT_CODE=$? mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports @@ -57,8 +58,9 @@ elif [[ ${COMPONENT} == "serving" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - mvn --define skipTests=true --projects serving --batch-mode clean install - mvn --projects serving test + # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level + mvn --projects serving --batch-mode --define skipTests=true --define enforcer.skip=true clean install + mvn --projects serving --define enforcer.skip=true test TEST_EXIT_CODE=$? cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports @@ -68,8 +70,8 @@ elif [[ ${COMPONENT} == "java-sdk" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - mvn --define skipTests=true --projects sdk/java --batch-mode clean install - mvn --projects sdk/java test + mvn --projects sdk/java --batch-mode --define skipTests=true --define enforcer.skip=true clean install + mvn --projects --define enforcer.skip=true sdk/java test TEST_EXIT_CODE=$? cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports From 66d2f6f8edd904ad833a70d77fd1565ca6ae6f20 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 25 Oct 2019 00:05:04 +0800 Subject: [PATCH 12/23] Fix incorrect order for java-sdk test script --- .prow/scripts/run_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 2c36417d67d..2e72b5f5bd1 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -70,8 +70,9 @@ elif [[ ${COMPONENT} == "java-sdk" ]]; then .prow/scripts/prepare_maven_cache.sh \ --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ + # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level mvn --projects sdk/java --batch-mode --define skipTests=true --define enforcer.skip=true clean install - mvn --projects --define enforcer.skip=true sdk/java test + mvn --projects sdk/java --define enforcer.skip=true test TEST_EXIT_CODE=$? cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports From 42c8c5f94bea20f72c0cb5c0436d2b124acb74ab Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 25 Oct 2019 09:28:55 +0800 Subject: [PATCH 13/23] Install database and kafka dependencies for Feast end to end test --- .prow/config.yaml | 92 +++++++++++++++++++-------------------- .prow/scripts/run_test.sh | 21 +++++++++ ingestion/pom.xml | 12 ----- 3 files changed, 67 insertions(+), 58 deletions(-) diff --git a/.prow/config.yaml b/.prow/config.yaml index 4a21bece186..093b205fa1f 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -90,7 +90,7 @@ presubmits: always_run: true spec: containers: - - image: python:3.6 + - image: python:3.7 command: [".prow/scripts/run_test.sh", "--component", "python-sdk"] - name: test-golang-sdk @@ -101,53 +101,53 @@ presubmits: - image: golang:1.13 command: [".prow/scripts/run_test.sh", "--component", "golang-sdk"] - - name: test-end-to-end - decorate: true - always_run: true - spec: - volumes: - - name: docker-socket-volume - hostPath: - path: /var/run/docker.sock - type: File - - name: service-account - secret: - secretName: prow-service-account - nodeSelector: - os: ubuntu - containers: - - image: google/cloud-sdk - # securityContext and docker socket volume mounts are needed because we are building - # Docker images in this job - securityContext: - privileged: true - volumeMounts: - - name: docker-socket-volume - mountPath: /var/run/docker.sock - - name: service-account - mountPath: /etc/service-account - readOnly: true - command: - - bash - - -c - - | - export FEAST_HOME=${PWD} - export FEAST_IMAGE_REGISTRY=us.gcr.io - export FEAST_IMAGE_TAG=${PULL_PULL_SHA} - export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID} - export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5} - export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv - export KAFKA_BROKERS=10.128.0.201:9092 - export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5} + # - name: test-end-to-end + # decorate: true + # always_run: true + # spec: + # volumes: + # - name: docker-socket-volume + # hostPath: + # path: /var/run/docker.sock + # type: File + # - name: service-account + # secret: + # secretName: prow-service-account + # nodeSelector: + # os: ubuntu + # containers: + # - image: google/cloud-sdk + # # securityContext and docker socket volume mounts are needed because we are building + # # Docker images in this job + # securityContext: + # privileged: true + # volumeMounts: + # - name: docker-socket-volume + # mountPath: /var/run/docker.sock + # - name: service-account + # mountPath: /etc/service-account + # readOnly: true + # command: + # - bash + # - -c + # - | + # export FEAST_HOME=${PWD} + # export FEAST_IMAGE_REGISTRY=us.gcr.io + # export FEAST_IMAGE_TAG=${PULL_PULL_SHA} + # export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID} + # export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 + # export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 + # export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5} + # export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv + # export KAFKA_BROKERS=10.128.0.201:9092 + # export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5} - . .prow/scripts/prepare_integration_test.sh - .prow/scripts/install_feast_and_run_e2e_test.sh - TEST_EXIT_CODE=$? - .prow/scripts/cleanup_feast_installation.sh + # . .prow/scripts/prepare_integration_test.sh + # .prow/scripts/install_feast_and_run_e2e_test.sh + # TEST_EXIT_CODE=$? + # .prow/scripts/cleanup_feast_installation.sh - exit ${TEST_EXIT_CODE} + # exit ${TEST_EXIT_CODE} # TODO: do a release when a git tag is pushed # diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh index 2e72b5f5bd1..bdb2f710a38 100755 --- a/.prow/scripts/run_test.sh +++ b/.prow/scripts/run_test.sh @@ -94,6 +94,27 @@ elif [[ ${COMPONENT} == "golang-sdk" ]]; then go get -u github.com/jstemmer/go-junit-report cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml +elif [[ ${COMPONENT} == "end-to-end" ]]; then + + apt-get -qq update + apt-get -y install redis-server + apt-get -y install postgresql + service postgresql start + redis-server --daemonize yes + + wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz + cd kafka_2.12-2.3.0/ + nohup bin/zookeeper-server-start.sh -daemon config/zookeeper.properties > /dev/null 2>&1 & + sleep 5 + nohup bin/kafka-server-start.sh -daemon config/server.properties > /dev/null 2>&1 & + sleep 5 + + .prow/scripts/prepare_maven_cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ + + mvn --batch-mode --define skipTests=true clean package + + else usage; exit 1 fi diff --git a/ingestion/pom.xml b/ingestion/pom.xml index b7e8033c23a..8811613ede3 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -86,18 +86,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - From 5bb25b34b04f9a485336454a9acc9c6a2a97e053 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 26 Oct 2019 16:55:12 +0800 Subject: [PATCH 14/23] Update Python SDK to not set source in FeatureSet when using default source Update in progress script for integration test: TODO.sh --- .prow/scripts/TODO.sh | 42 ++++++++++++++++++++++++++++++ protos/feast/core/FeatureSet.proto | 3 ++- sdk/python/feast/feature_set.py | 4 +-- tests/e2e/test_e2e.py | 17 +++++++++++- 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 .prow/scripts/TODO.sh diff --git a/.prow/scripts/TODO.sh b/.prow/scripts/TODO.sh new file mode 100644 index 00000000000..50ffd062d60 --- /dev/null +++ b/.prow/scripts/TODO.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -o pipefail +set -e + +apt-get -qq update + +apt-get -y install redis-server wget +redis-server --daemonize yes +redis-cli ping + +apt-get -y install postgresql +service postgresql start + +cat < /tmp/update-postgres-role.sh +psql -c "ALTER USER postgres PASSWORD 'password';" +EOF +chmod +x /tmp/update-postgres-role.sh +su -s /bin/bash -c /tmp/update-postgres-role.sh postgres +export PGPASSWORD=password +pg_isready + +wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz +cd kafka_2.12-2.3.0/ +nohup bin/zookeeper-server-start.sh -daemon config/zookeeper.properties > /var/log/zooker.log 2>&1 & +sleep 5 +nohup bin/kafka-server-start.sh -daemon config/server.properties > /var/log/kafka.log 2>&1 & +sleep 5 + +cd .. + + + +mvn --batch-mode --define skipTests=true clean package +java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/core.application.yml + +java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/serving.online.application.yml + + +wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index 24939055fc2..a80ae36f088 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -47,7 +47,8 @@ message FeatureSetSpec { // as nulls and indicated to end user google.protobuf.Duration max_age = 5; - // Source on which feature rows can be found + // Optional. Source on which feature rows can be found. + // If not set, source will be set to the default value configured in Feast Core. Source source = 6; } diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 87b95684d03..3760300056a 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -69,7 +69,7 @@ def __init__( if entities is not None: self.entities = entities if source is None: - self._source = KafkaSource() + self._source = None else: self._source = source self._max_age = max_age @@ -504,7 +504,7 @@ def to_proto(self) -> FeatureSetSpecProto: name=self.name, version=self.version, max_age=self.max_age, - source=self.source.to_proto(), + source=self.source.to_proto() if self.source is not None else None, features=[ field.to_proto() for field in self._fields.values() diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index e602ef89369..8e909005085 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -71,6 +71,19 @@ def test_basic(client): # Register feature set client.apply(cust_trans_fs) + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(3) + cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) + + if cust_trans_fs is None: + raise Exception( + "Client cannot retrieve 'customer_transactions' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) + cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) offset = random.randint(1000, 100000) # ensure a unique key space is used @@ -88,6 +101,8 @@ def test_basic(client): # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -103,8 +118,8 @@ def test_basic(client): "customer_transactions:1:total_transactions", ], ) # type: GetOnlineFeaturesResponse + if response is None: - time.sleep(1) continue returned_daily_transactions = float( From 48a71391937f768a6d98e7c264627468a22c0eaa Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 11:40:35 +0800 Subject: [PATCH 15/23] Fix Python SDK type mapping - float64 should map to ValueType.DOUBLE instead of ValueType.FLOAT - Ensure Python SDK installation use protobuf v3.10 from pypi. Earlier version of protobuf do not support accessing Enum value in Proto object directly. - Ensure version is passed as INT in e2e test. Set default allow_dirty to true so it's easier to debug error when running e2e test locally. --- sdk/python/feast/type_map.py | 4 ++-- sdk/python/setup.py | 2 +- tests/e2e/conftest.py | 4 ++-- tests/e2e/test_e2e.py | 20 +++++++++++++++----- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 30a97169e4d..d48acdf3842 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -106,7 +106,7 @@ def dtype_to_value_type(dtype): # TODO: to pass test_importer def pandas_dtype_to_feast_value_type(dtype: pd.DataFrame.dtypes) -> ValueType: type_map = { - "float64": ValueType.FLOAT, + "float64": ValueType.DOUBLE, "float32": ValueType.FLOAT, "int64": ValueType.INT64, "uint64": ValueType.INT64, @@ -247,7 +247,7 @@ def pd_value_to_proto_value(feast_value_type, value) -> ProtoValue: return ProtoValue(float_val=float(value)) elif feast_value_type == ValueType.DOUBLE: assert type(value) is float - return ProtoValue(float_val=value) + return ProtoValue(double_val=value) elif feast_value_type == ValueType.STRING: return ProtoValue(string_val=str(value)) elif feast_value_type == ValueType.BYTES: diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b78cb8a6c86..24f9c3f2402 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -36,7 +36,7 @@ "grpcio==1.*", "pandas==0.*", "pandavro==1.5.1", - "protobuf==3.*", + "protobuf==3.10.*", "PyYAML==5.1.2", "fastavro==0.*", "kafka-python==1.4.*", diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 6d862ec32a5..60c0076bce4 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,4 +1,4 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") - parser.addoption("--serving_url", action="store", default="localhost:6565") - parser.addoption("--allow_dirty", action="store", default="false") + parser.addoption("--serving_url", action="store", default="localhost:6566") + parser.addoption("--allow_dirty", action="store", default="true") diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 8e909005085..20c64644b04 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -84,8 +84,6 @@ def test_basic(client): "to be committed." ) - cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) - offset = random.randint(1000, 100000) # ensure a unique key space is used customer_data = pd.DataFrame( { @@ -139,7 +137,7 @@ def test_basic(client): @pytest.mark.timeout(300) def test_all_types(client): - all_types_fs = client.get_feature_set(name="all_types", version="1") + all_types_fs = client.get_feature_set(name="all_types", version=1) if all_types_fs is None: # Register new feature set if it doesnt exist @@ -167,7 +165,7 @@ def test_all_types(client): # Register feature set client.apply(all_types_fs) - all_types_fs = client.get_feature_set(name="all_types", version="1") + all_types_fs = client.get_feature_set(name="all_types", version=1) all_types_df = pd.DataFrame( { @@ -283,10 +281,21 @@ def test_large_volume(client): # Register feature set client.apply(cust_trans_fs) + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(3) cust_trans_fs = client.get_feature_set( name="customer_transactions_large", version=1 ) + if cust_trans_fs is None: + raise Exception( + "Client cannot retrieve 'customer_transactions' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) + offset = random.randint(1000000, 10000000) # ensure a unique key space customer_data = pd.DataFrame( { @@ -304,6 +313,8 @@ def test_large_volume(client): # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -321,7 +332,6 @@ def test_large_volume(client): ) # type: GetOnlineFeaturesResponse if response is None: - time.sleep(1) continue returned_daily_transactions = float( From 1756e68ea9e250d59b50c879cb25cda544d7185b Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 15:37:22 +0800 Subject: [PATCH 16/23] Refactor prow scripts such that every test is defined in a separate script - Since there are only 6 tests, there is not much benefit of combining tests into a single script Having each test defined on diff script file makes debugging easier --- .prow/config.yaml | 77 ++------ .prow/scripts/TODO.sh | 42 ----- .prow/scripts/cleanup_feast_installation.sh | 6 - ...maven_cache.sh => download-maven-cache.sh} | 0 .../scripts/install_feast_and_run_e2e_test.sh | 60 ------- .prow/scripts/install_feast_sdk.sh | 9 - .prow/scripts/install_test_tools.sh | 24 --- .prow/scripts/prepare_integration_test.sh | 65 ------- .prow/scripts/run_test.sh | 122 ------------- .prow/scripts/test-core-ingestion.sh | 20 +++ .prow/scripts/test-end-to-end.sh | 165 ++++++++++++++++++ .prow/scripts/test-golang-sdk.sh | 15 ++ .prow/scripts/test-java-sdk.sh | 13 ++ .prow/scripts/test-python-sdk.sh | 11 ++ .prow/scripts/test-serving.sh | 17 ++ tests/e2e/test_e2e.py | 12 ++ 16 files changed, 266 insertions(+), 392 deletions(-) delete mode 100644 .prow/scripts/TODO.sh delete mode 100755 .prow/scripts/cleanup_feast_installation.sh rename .prow/scripts/{prepare_maven_cache.sh => download-maven-cache.sh} (100%) delete mode 100755 .prow/scripts/install_feast_and_run_e2e_test.sh delete mode 100755 .prow/scripts/install_feast_sdk.sh delete mode 100755 .prow/scripts/install_test_tools.sh delete mode 100755 .prow/scripts/prepare_integration_test.sh delete mode 100755 .prow/scripts/run_test.sh create mode 100755 .prow/scripts/test-core-ingestion.sh create mode 100755 .prow/scripts/test-end-to-end.sh create mode 100755 .prow/scripts/test-golang-sdk.sh create mode 100755 .prow/scripts/test-java-sdk.sh create mode 100755 .prow/scripts/test-python-sdk.sh create mode 100755 .prow/scripts/test-serving.sh diff --git a/.prow/config.yaml b/.prow/config.yaml index 093b205fa1f..c1305bf6ec2 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -21,7 +21,7 @@ plank: deck: tide_update_period: 1s spyglass: - size_limit: 100e+6 # 100MB + size_limit: 50e+6 # 50MB viewers: "started.json|finished.json": ["metadata"] "build-log.txt": ["buildlog"] @@ -54,20 +54,9 @@ presubmits: decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - image: maven:3.6-jdk-8 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_test.sh", "--component", "core-ingestion"] + command: [".prow/scripts/test-core-ingestion.sh"] - name: test-serving decorate: true @@ -75,7 +64,7 @@ presubmits: spec: containers: - image: maven:3.6-jdk-8 - command: [".prow/scripts/run_test.sh", "--component", "serving"] + command: [".prow/scripts/test-serving.sh"] - name: test-java-sdk decorate: true @@ -83,7 +72,7 @@ presubmits: spec: containers: - image: maven:3.6-jdk-8 - command: [".prow/scripts/run_test.sh", "--component", "java-sdk"] + command: [".prow/scripts/test-java-sdk.sh"] - name: test-python-sdk decorate: true @@ -91,7 +80,7 @@ presubmits: spec: containers: - image: python:3.7 - command: [".prow/scripts/run_test.sh", "--component", "python-sdk"] + command: [".prow/scripts/test-python-sdk.sh"] - name: test-golang-sdk decorate: true @@ -99,55 +88,15 @@ presubmits: spec: containers: - image: golang:1.13 - command: [".prow/scripts/run_test.sh", "--component", "golang-sdk"] + command: [".prow/scripts/test-golang-sdk.sh"] - # - name: test-end-to-end - # decorate: true - # always_run: true - # spec: - # volumes: - # - name: docker-socket-volume - # hostPath: - # path: /var/run/docker.sock - # type: File - # - name: service-account - # secret: - # secretName: prow-service-account - # nodeSelector: - # os: ubuntu - # containers: - # - image: google/cloud-sdk - # # securityContext and docker socket volume mounts are needed because we are building - # # Docker images in this job - # securityContext: - # privileged: true - # volumeMounts: - # - name: docker-socket-volume - # mountPath: /var/run/docker.sock - # - name: service-account - # mountPath: /etc/service-account - # readOnly: true - # command: - # - bash - # - -c - # - | - # export FEAST_HOME=${PWD} - # export FEAST_IMAGE_REGISTRY=us.gcr.io - # export FEAST_IMAGE_TAG=${PULL_PULL_SHA} - # export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID} - # export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - # export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - # export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5} - # export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv - # export KAFKA_BROKERS=10.128.0.201:9092 - # export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5} - - # . .prow/scripts/prepare_integration_test.sh - # .prow/scripts/install_feast_and_run_e2e_test.sh - # TEST_EXIT_CODE=$? - # .prow/scripts/cleanup_feast_installation.sh - - # exit ${TEST_EXIT_CODE} + - name: test-end-to-end + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-end-to-end.sh"] # TODO: do a release when a git tag is pushed # diff --git a/.prow/scripts/TODO.sh b/.prow/scripts/TODO.sh deleted file mode 100644 index 50ffd062d60..00000000000 --- a/.prow/scripts/TODO.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -o pipefail -set -e - -apt-get -qq update - -apt-get -y install redis-server wget -redis-server --daemonize yes -redis-cli ping - -apt-get -y install postgresql -service postgresql start - -cat < /tmp/update-postgres-role.sh -psql -c "ALTER USER postgres PASSWORD 'password';" -EOF -chmod +x /tmp/update-postgres-role.sh -su -s /bin/bash -c /tmp/update-postgres-role.sh postgres -export PGPASSWORD=password -pg_isready - -wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz -cd kafka_2.12-2.3.0/ -nohup bin/zookeeper-server-start.sh -daemon config/zookeeper.properties > /var/log/zooker.log 2>&1 & -sleep 5 -nohup bin/kafka-server-start.sh -daemon config/server.properties > /var/log/kafka.log 2>&1 & -sleep 5 - -cd .. - - - -mvn --batch-mode --define skipTests=true clean package -java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ - --spring.config.location=file:///tmp/core.application.yml - -java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ - --spring.config.location=file:///tmp/serving.online.application.yml - - -wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh diff --git a/.prow/scripts/cleanup_feast_installation.sh b/.prow/scripts/cleanup_feast_installation.sh deleted file mode 100755 index 78b7d83a327..00000000000 --- a/.prow/scripts/cleanup_feast_installation.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -bq -q rm -rf --dataset ${FEAST_WAREHOUSE_DATASET} -gsutil -q rm ${BATCH_IMPORT_DATA_GCS_PATH} -helm delete --purge $FEAST_RELEASE_NAME diff --git a/.prow/scripts/prepare_maven_cache.sh b/.prow/scripts/download-maven-cache.sh similarity index 100% rename from .prow/scripts/prepare_maven_cache.sh rename to .prow/scripts/download-maven-cache.sh diff --git a/.prow/scripts/install_feast_and_run_e2e_test.sh b/.prow/scripts/install_feast_and_run_e2e_test.sh deleted file mode 100755 index 029f2e3e31a..00000000000 --- a/.prow/scripts/install_feast_and_run_e2e_test.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "============================================================" -echo "Installing Feast Release" -echo "============================================================" - -helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml - -echo "============================================================" -echo "Testing Batch Import" -echo "============================================================" - -cd ${FEAST_HOME}/integration-tests/testdata - -feast apply entity entity_specs/entity_1.yaml -feast apply feature feature_specs/entity_1*.yaml -feast jobs run import_specs/batch_from_gcs.yaml --wait - -cd $FEAST_HOME/integration-tests - -python -m testutils.validate_feature_values \ - --entity_spec_file=testdata/entity_specs/entity_1.yaml \ - --feature_spec_files=testdata/feature_specs/entity_1*.yaml \ - --expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \ - --expected-serving-values-file=testdata/feature_values/serving_1.csv \ - --bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \ - --feast-serving-url=${FEAST_SERVING_URL} - -echo "============================================================" -echo "Testing Streaming Import" -echo "============================================================" - -cd $FEAST_HOME/integration-tests/testdata - -feast apply entity entity_specs/entity_2.yaml -feast apply feature feature_specs/entity_2*.yaml -feast jobs run import_specs/stream_from_kafka.yaml & - -IMPORT_JOB_PID=$! -sleep 20 - -cd $FEAST_HOME/integration-tests - -python -m testutils.kafka_producer \ - --bootstrap_servers=$KAFKA_BROKERS \ - --topic=$KAFKA_TOPICS \ - --entity_spec_file=testdata/entity_specs/entity_2.yaml \ - --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ - --feature_values_file=testdata/feature_values/ingestion_2.csv -sleep 20 - -python -m testutils.validate_feature_values \ - --entity_spec_file=testdata/entity_specs/entity_2.yaml \ - --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ - --expected-serving-values-file=testdata/feature_values/serving_2.csv \ - --feast-serving-url=$FEAST_SERVING_URL - -kill -9 ${IMPORT_JOB_PID} diff --git a/.prow/scripts/install_feast_sdk.sh b/.prow/scripts/install_feast_sdk.sh deleted file mode 100755 index 723199b9e51..00000000000 --- a/.prow/scripts/install_feast_sdk.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script ensures latest Feast Python SDK and Feast CLI are installed - -pip install -qe ${FEAST_HOME}/sdk/python -pip install -qr ${FEAST_HOME}/integration-tests/testutils/requirements.txt -go build -o /usr/local/bin/feast ./cli/feast &> /dev/null -feast config set coreURI ${FEAST_CORE_URL} diff --git a/.prow/scripts/install_test_tools.sh b/.prow/scripts/install_test_tools.sh deleted file mode 100755 index a2e30bbc3a2..00000000000 --- a/.prow/scripts/install_test_tools.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script installs the following Feast test utilities: -# ============================================================ -# - gettext package so we can use envsubst command to provide values to helm template file -# - Python 3.6 because Feast requires Python version 3.6 and above -# - Golang if we need to build Feast CLI from source -# - Helm if we want to install Feast release - -apt-get -qq update -apt-get -y install curl wget gettext &> /dev/null - -curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh -bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null -export PATH=/miniconda/bin:$PATH - -wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf - -mv go /usr/local/ -export PATH=/usr/local/go/bin:$PATH -export GO111MODULE=on - -wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz -mv linux-amd64/helm /usr/local/bin/helm diff --git a/.prow/scripts/prepare_integration_test.sh b/.prow/scripts/prepare_integration_test.sh deleted file mode 100755 index 1a08f26475b..00000000000 --- a/.prow/scripts/prepare_integration_test.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -set -e - -usage() -{ - echo "usage: prepare_integration_test.sh [--skip-build true]" -} - -while [ "$1" != "" ]; do - case "$1" in - --skip-build ) SKIP_BUILD=true; shift;; - * ) usage; exit 1 - esac - shift -done - -# Authenticate to Google Cloud and GKE -# ============================================================ -GOOGLE_PROJECT_ID=kf-feast -KUBE_CLUSTER_NAME=primary-test-cluster -KUBE_CLUSTER_ZONE=us-central1-a -KEY_FILE=/etc/service-account/service-account.json - -gcloud -q auth activate-service-account --key-file=${KEY_FILE} -gcloud -q auth configure-docker -gcloud -q config set project ${GOOGLE_PROJECT_ID} -gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID} -export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE} - -# Install Python 3.6, Golang 1.12, Helm and Feast SDK -# ============================================================ -. .prow/scripts/install_test_tools.sh -. .prow/scripts/install_feast_sdk.sh -.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME} - -# Prepare Feast test data and config -# ============================================================ - -bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET} -gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH} - -BUILD_ID=${BUILD_ID:0:5} -envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml -cd ${FEAST_HOME}/integration-tests/testdata/import_specs -envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml -envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml - -if [[ ! ${SKIP_BUILD} ]]; then - -echo "============================================================" -echo "Building Feast for Testing" -echo "============================================================" -cd ${FEAST_HOME} -docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . & -docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . & -wait -docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} & -docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} & -wait - -fi - -# Switch back context to original directory -set +ex -cd ${FEAST_HOME} \ No newline at end of file diff --git a/.prow/scripts/run_test.sh b/.prow/scripts/run_test.sh deleted file mode 100755 index bdb2f710a38..00000000000 --- a/.prow/scripts/run_test.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -# Default artifact location setting in Prow jobs -LOGS_ARTIFACT_PATH=/logs/artifacts - -set -o pipefail - -usage() -{ - echo "Run test on a Feast component. - -Usage: run_test.sh --component - - is one of: -- core-ingestion (core depends on ingestion so they are tested together) -- serving -- java-sdk -- python-sdk -- golang-sdk - -This script also runs commands before and after the main test task, such as: -- Download cached Maven packages for faster tests -- Saving the test output report so it can be viewed with Spyglass UI in Prow. - By default, the configured log path is "/logs" and test artifacts should - be saved to "/logs/artifacts" directory. -" -} - -while [ "$1" != "" ]; do - case "$1" in - --component ) COMPONENT="$2"; shift;; - * ) usage; exit 1 - esac - shift -done - -if [[ ! ${COMPONENT} ]]; then - usage; exit 1; -fi - -if [[ ${COMPONENT} == "core-ingestion" ]]; then - - .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - - # Core depends on Ingestion so they are tested together - # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level - mvn --projects core,ingestion --batch-mode --define skipTests=true --define enforcer.skip=true clean install - mvn --projects core,ingestion --define enforcer.skip=true test - TEST_EXIT_CODE=$? - - mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports - cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ - cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ - -elif [[ ${COMPONENT} == "serving" ]]; then - - .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - - # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level - mvn --projects serving --batch-mode --define skipTests=true --define enforcer.skip=true clean install - mvn --projects serving --define enforcer.skip=true test - TEST_EXIT_CODE=$? - - cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports - -elif [[ ${COMPONENT} == "java-sdk" ]]; then - - .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - - # Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level - mvn --projects sdk/java --batch-mode --define skipTests=true --define enforcer.skip=true clean install - mvn --projects sdk/java --define enforcer.skip=true test - TEST_EXIT_CODE=$? - - cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports - -elif [[ ${COMPONENT} == "python-sdk" ]]; then - - cd sdk/python - pip install -r requirements-test.txt - pip install . - pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml - TEST_EXIT_CODE=$? - -elif [[ ${COMPONENT} == "golang-sdk" ]]; then - - cd sdk/go - go test -v 2>&1 | tee /tmp/test_output - TEST_EXIT_CODE=$? - - go get -u github.com/jstemmer/go-junit-report - cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml - -elif [[ ${COMPONENT} == "end-to-end" ]]; then - - apt-get -qq update - apt-get -y install redis-server - apt-get -y install postgresql - service postgresql start - redis-server --daemonize yes - - wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz - cd kafka_2.12-2.3.0/ - nohup bin/zookeeper-server-start.sh -daemon config/zookeeper.properties > /dev/null 2>&1 & - sleep 5 - nohup bin/kafka-server-start.sh -daemon config/server.properties > /dev/null 2>&1 & - sleep 5 - - .prow/scripts/prepare_maven_cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/ - - mvn --batch-mode --define skipTests=true clean package - - -else - usage; exit 1 -fi - -exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-core-ingestion.sh b/.prow/scripts/test-core-ingestion.sh new file mode 100755 index 00000000000..98a47ca68c9 --- /dev/null +++ b/.prow/scripts/test-core-ingestion.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Core depends on Ingestion so they are tested together +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects core,ingestion --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects core,ingestion --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports +cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ +cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh new file mode 100755 index 00000000000..fcfeaf1b52d --- /dev/null +++ b/.prow/scripts/test-end-to-end.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +if ! cat /etc/*release | grep -q stretch; then + echo ${BASH_SOURCE} only supports Debian stretch. + echo Please change your operating system to use this script. + exit 1 +fi + +echo " +This script will run end-to-end tests for Feast Online Serving. + +1. Install Redis as the store for Feast Online Serving. +2. Install Postgres for persisting Feast metadata. +3. Install Kafka and Zookeeper as the Source in Feast. +4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from + tests/e2e via pytest. +" + +# Install Redis at localhost:6379 +apt-get -qq update +apt-get -y install redis-server wget +redis-server --daemonize yes +redis-cli ping + +# Install Postgres at localhost:5432 +# Initialize with database 'postgres', user 'postgres', password 'password' +apt-get -y install postgresql +service postgresql start +cat < /tmp/update-postgres-role.sh +psql -c "ALTER USER postgres PASSWORD 'password';" +EOF +chmod +x /tmp/update-postgres-role.sh +su -s /bin/bash -c /tmp/update-postgres-role.sh postgres +export PGPASSWORD=password +pg_isready + +# Install Zookeeper at localhost:2181 +# Install Kafka at localhost:9092 +wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz +mv kafka_2.12-2.3.0/ /tmp/kafka +nohup /tmp/kafka/bin/zookeeper-server-start.sh -daemon /tmp/kafka/config/zookeeper.properties > /var/log/zooker.log 2>&1 & +sleep 5 +nohup /tmp/kafka/bin/kafka-server-start.sh -daemon /tmp/kafka/config/server.properties > /var/log/kafka.log 2>&1 & +sleep 5 + +# Build jars for Feast +mvn --batch-mode --define skipTests=true clean package + +# Start Feast Core in background +cat < /tmp/core.application.yml +grpc: + port: 6565 + enable-reflection: true + +feast: + version: 0.3 + jobs: + runner: DirectRunner + options: {} + metrics: + enabled: false + + stream: + type: kafka + options: + bootstrapServers: localhost:9092 + replicationFactor: 1 + partitions: 1 + +spring: + jpa: + properties.hibernate.format_sql: true + hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl + hibernate.ddl-auto: update + datasource: + url: jdbc:postgresql://localhost:5432/postgres + username: postgres + password: password + +management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: false +EOF + +nohup java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/core.application.yml \ + &> /var/log/feast-core.log & +sleep 20 +tail -n50 /var/log/feast-core.log + +# Start Feast Online Serving in background +cat < /tmp/serving.store.redis.yml +name: serving +type: REDIS +redis_config: + host: localhost + port: 6379 +subscriptions: + - name: .* + version: ">0" +EOF + +cat < /tmp/serving.online.application.yml +feast: + version: 0.3 + core-host: localhost + core-grpc-port: 6565 + + tracing: + enabled: false + + store: + config-path: /tmp/serving.store.redis.yml + redis-pool-max-size: 128 + redis-pool-max-idle: 16 + + jobs: + staging-location: gs://feast-templocation-kf-feast/staging-location + store-type: + store-options: {} + +grpc: + port: 6566 + enable-reflection: true + +spring: + main: + web-environment: false +EOF + +nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/serving.online.application.yml \ + &> /var/log/feast-serving-online.log & +sleep 15 +tail -n50 /var/log/feast-serving-online.log + +# Install Python 3.7 with Miniconda +wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh +bash /tmp/miniconda.sh -b -p /root/miniconda -f +/root/miniconda/bin/conda init +source ~/.bashrc + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +# Run end-to-end tests with pytest with the Python SDK +pip install sdk/feast +pip install -r tests/e2e/requirements.txt + +ORIGINAL_DIR=$(pwd) + +cd tests/e2e +set +e +pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +TEST_EXIT_CODE=$? + +cd ${ORIGINAL_DIR} +exit ${TEST_EXIT_CODE} diff --git a/.prow/scripts/test-golang-sdk.sh b/.prow/scripts/test-golang-sdk.sh new file mode 100755 index 00000000000..b586927a512 --- /dev/null +++ b/.prow/scripts/test-golang-sdk.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -o pipefail + +cd sdk/go +go test -v 2>&1 | tee /tmp/test_output +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +go get -u github.com/jstemmer/go-junit-report +cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-java-sdk.sh b/.prow/scripts/test-java-sdk.sh new file mode 100755 index 00000000000..0731b77976f --- /dev/null +++ b/.prow/scripts/test-java-sdk.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects sdk/java --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects sdk/java --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-python-sdk.sh b/.prow/scripts/test-python-sdk.sh new file mode 100755 index 00000000000..eb40f921c7b --- /dev/null +++ b/.prow/scripts/test-python-sdk.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +cd sdk/python +pip install -r requirements-test.txt +pip install . +pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/.prow/scripts/test-serving.sh b/.prow/scripts/test-serving.sh new file mode 100755 index 00000000000..d105f733827 --- /dev/null +++ b/.prow/scripts/test-serving.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects serving --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects serving --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 20c64644b04..9d5eff14f96 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -165,8 +165,20 @@ def test_all_types(client): # Register feature set client.apply(all_types_fs) + + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(3) all_types_fs = client.get_feature_set(name="all_types", version=1) + if all_types_fs is None: + raise Exception( + "Client cannot retrieve 'all_types_fs' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) + all_types_df = pd.DataFrame( { "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(3)], From 546b215107dcd0d5a79d854c83e0d6643f99a265 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 15:39:37 +0800 Subject: [PATCH 17/23] Remove reference to actual project id --- infra/charts/feast/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index c879363611c..be3b4138290 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -205,8 +205,9 @@ warehouse-serving: name: warehouse type: BIGQUERY bigquery_config: - projectId: the-big-data-staging-007 - datasetId: feast + # Replace with your Google Cloud project configuration + projectId: google-project-id + datasetId: bigquery-dataset subscriptions: - name: .* version: ">0" From 85f328215d4ed91b56a858085225e9c5d3b69931 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 15:55:55 +0800 Subject: [PATCH 18/23] Log current stage in end to end test script - Update test case DirectRunnerJobManagerTest.shouldStartDirectJobAndRegisterPipelineResult to set project option to empty string for DirectRunner - Update /usr/sbin/policy-rc.d used in Maven Docker image, so Redis installation will succeed --- .prow/scripts/test-end-to-end.sh | 49 +++++++++++++++++-- .../direct/DirectRunnerJobManagerTest.java | 1 + 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index fcfeaf1b52d..1d22ed2a450 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -19,16 +19,26 @@ This script will run end-to-end tests for Feast Online Serving. tests/e2e via pytest. " -# Install Redis at localhost:6379 +echo " +============================================================ +Installing Redis at localhost:6379 +============================================================ +" apt-get -qq update +# Allow starting serving in this Maven Docker image. Default set to not allowed. +echo "exit 0" > /usr/sbin/policy-rc.d apt-get -y install redis-server wget redis-server --daemonize yes redis-cli ping -# Install Postgres at localhost:5432 -# Initialize with database 'postgres', user 'postgres', password 'password' +echo " +============================================================ +Installing Postgres at localhost:5432 +============================================================ +" apt-get -y install postgresql service postgresql start +# Initialize with database: 'postgres', user: 'postgres', password: 'password' cat < /tmp/update-postgres-role.sh psql -c "ALTER USER postgres PASSWORD 'password';" EOF @@ -37,8 +47,12 @@ su -s /bin/bash -c /tmp/update-postgres-role.sh postgres export PGPASSWORD=password pg_isready -# Install Zookeeper at localhost:2181 -# Install Kafka at localhost:9092 +echo " +============================================================ +Installing Zookeeper at localhost:2181 +Installing Kafka at localhost:9092 +============================================================ +" wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz mv kafka_2.12-2.3.0/ /tmp/kafka nohup /tmp/kafka/bin/zookeeper-server-start.sh -daemon /tmp/kafka/config/zookeeper.properties > /var/log/zooker.log 2>&1 & @@ -46,9 +60,19 @@ sleep 5 nohup /tmp/kafka/bin/kafka-server-start.sh -daemon /tmp/kafka/config/server.properties > /var/log/kafka.log 2>&1 & sleep 5 +echo " +============================================================ +Building jars for Feast +============================================================ +" # Build jars for Feast mvn --batch-mode --define skipTests=true clean package +echo " +============================================================ +Starting Feast Core +============================================================ +" # Start Feast Core in background cat < /tmp/core.application.yml grpc: @@ -95,6 +119,11 @@ nohup java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ sleep 20 tail -n50 /var/log/feast-core.log +echo " +============================================================ +Starting Feast Online Serving +============================================================ +" # Start Feast Online Serving in background cat < /tmp/serving.store.redis.yml name: serving @@ -141,6 +170,11 @@ nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ sleep 15 tail -n50 /var/log/feast-serving-online.log +echo " +============================================================ +Installing Python 3.7 with Miniconda and Feast SDK +============================================================ +" # Install Python 3.7 with Miniconda wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh bash /tmp/miniconda.sh -b -p /root/miniconda -f @@ -156,6 +190,11 @@ pip install -r tests/e2e/requirements.txt ORIGINAL_DIR=$(pwd) +echo " +============================================================ +Running end-to-end tests with pytest at 'tests/e2e' +============================================================ +" cd tests/e2e set +e pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java index 93f2519c383..ad58bf2f300 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java +++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java @@ -73,6 +73,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { expectedPipelineOptions.setAppName("DirectRunnerJobManager"); expectedPipelineOptions.setRunner(DirectRunner.class); expectedPipelineOptions.setBlockOnRun(false); + expectedPipelineOptions.setProject(""); expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store))); expectedPipelineOptions .setFeatureSetSpecJson(Lists.newArrayList(printer.print(featureSetSpec))); From 679ff120dc652cf94e82433eb7e481f71c36de91 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 16:02:37 +0800 Subject: [PATCH 19/23] Fix missing option to specify miniconda download target - Also use cached maven packages when running e2e test --- .prow/scripts/test-end-to-end.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 1d22ed2a450..dfe71bb2bb7 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -65,6 +65,11 @@ echo " Building jars for Feast ============================================================ " + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + # Build jars for Feast mvn --batch-mode --define skipTests=true clean package @@ -176,7 +181,8 @@ Installing Python 3.7 with Miniconda and Feast SDK ============================================================ " # Install Python 3.7 with Miniconda -wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh +wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \ + -O /tmp/miniconda.sh bash /tmp/miniconda.sh -b -p /root/miniconda -f /root/miniconda/bin/conda init source ~/.bashrc From 580636648dc5c2442d5fc20d38f11d36a30c5012 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 16:37:29 +0800 Subject: [PATCH 20/23] Fix incorrect path to installing Python SDK --- .prow/scripts/test-end-to-end.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index dfe71bb2bb7..78ed402ae00 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -55,10 +55,12 @@ Installing Kafka at localhost:9092 " wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz mv kafka_2.12-2.3.0/ /tmp/kafka -nohup /tmp/kafka/bin/zookeeper-server-start.sh -daemon /tmp/kafka/config/zookeeper.properties > /var/log/zooker.log 2>&1 & +nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & sleep 5 -nohup /tmp/kafka/bin/kafka-server-start.sh -daemon /tmp/kafka/config/server.properties > /var/log/kafka.log 2>&1 & +tail -n20 /var/log/zookeeper.log +nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & sleep 5 +tail -n20 /var/log/kafka.log echo " ============================================================ @@ -122,7 +124,7 @@ nohup java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 20 -tail -n50 /var/log/feast-core.log +tail -n20 /var/log/feast-core.log echo " ============================================================ @@ -173,7 +175,7 @@ nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.online.application.yml \ &> /var/log/feast-serving-online.log & sleep 15 -tail -n50 /var/log/feast-serving-online.log +tail -n20 /var/log/feast-serving-online.log echo " ============================================================ @@ -191,7 +193,7 @@ source ~/.bashrc LOGS_ARTIFACT_PATH=/logs/artifacts # Run end-to-end tests with pytest with the Python SDK -pip install sdk/feast +pip install sdk/python pip install -r tests/e2e/requirements.txt ORIGINAL_DIR=$(pwd) From a958eda4ae9aeff4552df0f4089fbea091b38812 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 17:05:24 +0800 Subject: [PATCH 21/23] Add small wait after ingestion, before validating the saved features, in feast e2e test --- .prow/scripts/test-end-to-end.sh | 18 +++++++++--------- tests/e2e/test_e2e.py | 4 +++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 78ed402ae00..ab7ba22bb35 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -73,7 +73,7 @@ Building jars for Feast --output-dir /root/ # Build jars for Feast -mvn --batch-mode --define skipTests=true clean package +mvn --quiet --batch-mode --define skipTests=true clean package echo " ============================================================ @@ -124,7 +124,7 @@ nohup java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 20 -tail -n20 /var/log/feast-core.log +tail -n10 /var/log/feast-core.log echo " ============================================================ @@ -175,7 +175,7 @@ nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.online.application.yml \ &> /var/log/feast-serving-online.log & sleep 15 -tail -n20 /var/log/feast-serving-online.log +tail -n10 /var/log/feast-serving-online.log echo " ============================================================ @@ -189,21 +189,21 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f /root/miniconda/bin/conda init source ~/.bashrc -# Default artifact location setting in Prow jobs -LOGS_ARTIFACT_PATH=/logs/artifacts - -# Run end-to-end tests with pytest with the Python SDK +# Install Feast Python SDK and test requirements pip install sdk/python pip install -r tests/e2e/requirements.txt -ORIGINAL_DIR=$(pwd) - echo " ============================================================ Running end-to-end tests with pytest at 'tests/e2e' ============================================================ " +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +ORIGINAL_DIR=$(pwd) cd tests/e2e + set +e pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 9d5eff14f96..f92167a7b52 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -230,9 +230,12 @@ def test_all_types(client): # Ingest user embedding data all_types_fs.ingest(dataframe=all_types_df) + time.sleep(3) # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -258,7 +261,6 @@ def test_all_types(client): ) # type: GetOnlineFeaturesResponse if response is None: - time.sleep(1) continue returned_float_list = ( From 23bb84f380e8ecab7a76d792167bfad86d1a8f98 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 17:19:01 +0800 Subject: [PATCH 22/23] Increase wait time before checking, after applying feature set --- .prow/scripts/test-end-to-end.sh | 10 +++++----- tests/e2e/test_e2e.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index ab7ba22bb35..1a90eb90ad2 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -36,7 +36,7 @@ echo " Installing Postgres at localhost:5432 ============================================================ " -apt-get -y install postgresql +apt-get -y install postgresql > /var/log/postgresql.install.log service postgresql start # Initialize with database: 'postgres', user: 'postgres', password: 'password' cat < /tmp/update-postgres-role.sh @@ -57,10 +57,10 @@ wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar mv kafka_2.12-2.3.0/ /tmp/kafka nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & sleep 5 -tail -n20 /var/log/zookeeper.log +tail -n10 /var/log/zookeeper.log nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & sleep 5 -tail -n20 /var/log/kafka.log +tail -n10 /var/log/kafka.log echo " ============================================================ @@ -190,8 +190,8 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f source ~/.bashrc # Install Feast Python SDK and test requirements -pip install sdk/python -pip install -r tests/e2e/requirements.txt +pip install -q sdk/python +pip install -qr tests/e2e/requirements.txt echo " ============================================================ diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index f92167a7b52..a529dd1a24b 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -73,7 +73,7 @@ def test_basic(client): # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset - time.sleep(3) + time.sleep(15) cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) if cust_trans_fs is None: @@ -168,7 +168,7 @@ def test_all_types(client): # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset - time.sleep(3) + time.sleep(10) all_types_fs = client.get_feature_set(name="all_types", version=1) if all_types_fs is None: @@ -297,7 +297,7 @@ def test_large_volume(client): # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset - time.sleep(3) + time.sleep(10) cust_trans_fs = client.get_feature_set( name="customer_transactions_large", version=1 ) From 675c32a4abfef7ec3cb46fb8e705046447273b25 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sun, 27 Oct 2019 22:42:48 +0800 Subject: [PATCH 23/23] Make log cleaner --- .prow/scripts/install_google_cloud_sdk.sh | 2 +- .prow/scripts/test-end-to-end.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.prow/scripts/install_google_cloud_sdk.sh b/.prow/scripts/install_google_cloud_sdk.sh index 7684760dee9..c6356557cec 100755 --- a/.prow/scripts/install_google_cloud_sdk.sh +++ b/.prow/scripts/install_google_cloud_sdk.sh @@ -30,7 +30,7 @@ KUBE_CLUSTER_ZONE=us-central1-a curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C / export PATH=/google-cloud-sdk/bin:${PATH} -gcloud -q components install kubectl +gcloud -q components install kubectl &> /var/log/kubectl.install.log if [[ ${KEY_FILE} ]]; then gcloud -q auth activate-service-account --key-file=${KEY_FILE} diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 1a90eb90ad2..89ddd826e33 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -10,7 +10,7 @@ if ! cat /etc/*release | grep -q stretch; then fi echo " -This script will run end-to-end tests for Feast Online Serving. +This script will run end-to-end tests for Feast Core and Online Serving. 1. Install Redis as the store for Feast Online Serving. 2. Install Postgres for persisting Feast metadata. @@ -27,7 +27,7 @@ Installing Redis at localhost:6379 apt-get -qq update # Allow starting serving in this Maven Docker image. Default set to not allowed. echo "exit 0" > /usr/sbin/policy-rc.d -apt-get -y install redis-server wget +apt-get -y install redis-server wget > /var/log/redis.install.log redis-server --daemonize yes redis-cli ping