Skip to content

Commit a27abcd

Browse files
davidheryantofeast-ci-bot
davidheryanto
authored and
feast-ci-bot
committed
Update Prow configuration for Feast CI (#197)
* Add .prow folder to store prow config Remove travis * Add scripts for unit tests and integration test
1 parent 88d32ad commit a27abcd

27 files changed

+475
-275
lines changed

‎.prow/config.yaml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
prowjob_namespace: default
2+
pod_namespace: test-pods
3+
4+
plank:
5+
allow_cancellations: true
6+
job_url_prefix: http://prow.feast.ai/view/gcs
7+
report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})'
8+
default_decoration_config:
9+
timeout: 7200000000000 # 2h
10+
grace_period: 15000000000 # 15s
11+
utility_images:
12+
clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
13+
initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
14+
entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
15+
sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
16+
gcs_configuration:
17+
bucket: feast-templocation-kf-feast
18+
path_strategy: explicit
19+
gcs_credentials_secret: prow-service-account
20+
21+
deck:
22+
tide_update_period: 1s
23+
spyglass:
24+
size_limit: 100e+6 # 100MB
25+
viewers:
26+
"started.json|finished.json": ["metadata"]
27+
"build-log.txt": ["buildlog"]
28+
"report.xml": ["junit"]
29+
"artifacts/.*\\.xml": ["junit"]
30+
"surefire-reports/.*\\.xml": ["junit"]
31+
32+
tide:
33+
queries:
34+
- repos:
35+
- gojek/feast
36+
labels:
37+
- lgtm
38+
- approved
39+
missingLabels:
40+
- do-not-merge
41+
- do-not-merge/hold
42+
- do-not-merge/invalid-owners-file
43+
- do-not-merge/work-in-progress
44+
- needs-rebase
45+
merge_method:
46+
gojek/feast: squash
47+
blocker_label: merge-blocker
48+
squash_label: tide/squash
49+
50+
# presubmits list Prow jobs that run on pull requests
51+
presubmits:
52+
gojek/feast:
53+
- name: unit-test-core
54+
decorate: true
55+
always_run: true
56+
spec:
57+
containers:
58+
- image: maven:3.6-jdk-8
59+
command: [".prow/scripts/run_unit_test.sh", "--component", "core"]
60+
61+
- name: unit-test-ingestion
62+
decorate: true
63+
always_run: true
64+
spec:
65+
containers:
66+
- image: maven:3.6-jdk-8
67+
command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"]
68+
69+
- name: unit-test-serving
70+
decorate: true
71+
always_run: true
72+
spec:
73+
containers:
74+
- image: maven:3.6-jdk-8
75+
command: [".prow/scripts/run_unit_test.sh", "--component", "serving"]
76+
77+
- name: unit-test-cli
78+
decorate: true
79+
always_run: true
80+
spec:
81+
containers:
82+
- image: golang:1.12
83+
env:
84+
- name: GO111MODULE
85+
value: "on"
86+
command: [".prow/scripts/run_unit_test.sh", "--component", "cli"]
87+
88+
- name: integration-test
89+
decorate: true
90+
always_run: true
91+
spec:
92+
volumes:
93+
- name: docker-socket-volume
94+
hostPath:
95+
path: /var/run/docker.sock
96+
type: File
97+
- name: service-account
98+
secret:
99+
secretName: prow-service-account
100+
nodeSelector:
101+
os: ubuntu
102+
containers:
103+
- image: google/cloud-sdk
104+
# securityContext and docker socket vol mounts are needed because we are building
105+
# Docker images in this job
106+
securityContext:
107+
privileged: true
108+
volumeMounts:
109+
- name: docker-socket-volume
110+
mountPath: /var/run/docker.sock
111+
- name: service-account
112+
mountPath: /etc/service-account
113+
readOnly: true
114+
command:
115+
- bash
116+
- -c
117+
- |
118+
export FEAST_HOME=${PWD}
119+
export FEAST_IMAGE_REGISTRY=us.gcr.io
120+
export FEAST_IMAGE_TAG=${PULL_PULL_SHA}
121+
export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID:0:5}
122+
export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
123+
export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
124+
export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5}
125+
export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv
126+
export KAFKA_BROKERS=10.128.0.201:9092
127+
export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5}
128+
129+
. .prow/scripts/prepare_integration_test.sh
130+
.prow/scripts/install_feast_and_run_e2e_test.sh
131+
TEST_EXIT_CODE=$?
132+
.prow/scripts/cleanup_feast_installation.sh
133+
134+
exit ${TEST_EXIT_CODE}
135+
136+
# TODO: do a release when a git tag is pushed
137+
# postsubmits list Prow jobs that run on every push
138+
#
139+
# postsubmits:
140+
# gojek/feast:

‎.prow/plugins.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
plugins:
2+
gojek/feast:
3+
- approve
4+
- assign
5+
- help
6+
- hold
7+
- label
8+
- lgtm
9+
- lifecycle
10+
- size
11+
- verify-owners
12+
- wip
13+
- trigger
14+
- config-updater
15+
16+
config_updater:
17+
maps:
18+
.prow/config.yaml:
19+
name: config
20+
21+
external_plugins:
22+
gojek/feast:
23+
- name: needs-rebase
24+
events:
25+
- pull_request

‎.prow/scripts/cleanup_feast_installation.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
bq -q rm -rf --dataset ${FEAST_WAREHOUSE_DATASET}
5+
gsutil -q rm ${BATCH_IMPORT_DATA_GCS_PATH}
6+
helm delete --purge $FEAST_RELEASE_NAME

‎.prow/scripts/install_feast_and_run_e2e_test.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
echo "============================================================"
6+
echo "Installing Feast Release"
7+
echo "============================================================"
8+
9+
helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml
10+
11+
echo "============================================================"
12+
echo "Testing Batch Import"
13+
echo "============================================================"
14+
15+
cd ${FEAST_HOME}/integration-tests/testdata
16+
17+
feast apply entity entity_specs/entity_1.yaml
18+
feast apply feature feature_specs/entity_1*.yaml
19+
feast jobs run import_specs/batch_from_gcs.yaml --wait
20+
21+
cd $FEAST_HOME/integration-tests
22+
23+
python -m testutils.validate_feature_values \
24+
--entity_spec_file=testdata/entity_specs/entity_1.yaml \
25+
--feature_spec_files=testdata/feature_specs/entity_1*.yaml \
26+
--expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
27+
--expected-serving-values-file=testdata/feature_values/serving_1.csv \
28+
--bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \
29+
--feast-serving-url=${FEAST_SERVING_URL}
30+
31+
echo "============================================================"
32+
echo "Testing Streaming Import"
33+
echo "============================================================"
34+
35+
cd $FEAST_HOME/integration-tests/testdata
36+
37+
feast apply entity entity_specs/entity_2.yaml
38+
feast apply feature feature_specs/entity_2*.yaml
39+
feast jobs run import_specs/stream_from_kafka.yaml &
40+
41+
IMPORT_JOB_PID=$!
42+
sleep 20
43+
44+
cd $FEAST_HOME/integration-tests
45+
46+
python -m testutils.kafka_producer \
47+
--bootstrap_servers=$KAFKA_BROKERS \
48+
--topic=$KAFKA_TOPICS \
49+
--entity_spec_file=testdata/entity_specs/entity_2.yaml \
50+
--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
51+
--feature_values_file=testdata/feature_values/ingestion_2.csv
52+
sleep 20
53+
54+
python -m testutils.validate_feature_values \
55+
--entity_spec_file=testdata/entity_specs/entity_2.yaml \
56+
--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
57+
--expected-serving-values-file=testdata/feature_values/serving_2.csv \
58+
--feast-serving-url=$FEAST_SERVING_URL
59+
60+
kill -9 ${IMPORT_JOB_PID}

‎.prow/scripts/install_feast_sdk.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
# This script ensures latest Feast Python SDK and Feast CLI are installed
5+
6+
pip install -qe ${FEAST_HOME}/sdk/python
7+
pip install -qr ${FEAST_HOME}/integration-tests/testutils/requirements.txt
8+
go build -o /usr/local/bin/feast ./cli/feast &> /dev/null
9+
feast config set coreURI ${FEAST_CORE_URL}

‎.prow/scripts/install_google_cloud_sdk.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
usage()
5+
{
6+
echo "usage: . install_google_cloud_sdk.sh
7+
[--with-key-file local file path to service account json]
8+
9+
NOTE: requires 'dot' before install_google_cloud_sdk.sh
10+
so that the PATH variable is exported succesfully to
11+
the calling process, i.e. you don't need to provide
12+
full path to gcloud command after installation
13+
14+
--with-key-file is optional,
15+
if no authentication is required"
16+
}
17+
18+
while [ "$1" != "" ]; do
19+
case "$1" in
20+
--with-key-file ) KEY_FILE="$2"; shift;;
21+
* ) usage; exit 1
22+
esac
23+
shift
24+
done
25+
26+
GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
27+
GOOGLE_PROJECT_ID=kf-feast
28+
KUBE_CLUSTER_NAME=primary-test-cluster
29+
KUBE_CLUSTER_ZONE=us-central1-a
30+
31+
curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C /
32+
export PATH=/google-cloud-sdk/bin:${PATH}
33+
gcloud -q components install kubectl
34+
35+
if [[ ${KEY_FILE} ]]; then
36+
gcloud -q auth activate-service-account --key-file=${KEY_FILE}
37+
gcloud -q auth configure-docker
38+
gcloud -q config set project ${GOOGLE_PROJECT_ID}
39+
gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID}
40+
export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE}
41+
fi

‎.prow/scripts/install_test_tools.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
# This script installs the following Feast test utilities:
5+
# ============================================================
6+
# - gettext package so we can use envsubst command to provide values to helm template file
7+
# - Python 3.6 because Feast requires Python version 3.6 and above
8+
# - Golang if we need to build Feast CLI from source
9+
# - Helm if we want to install Feast release
10+
11+
apt-get -qq update
12+
apt-get -y install curl wget gettext &> /dev/null
13+
14+
curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh
15+
bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null
16+
export PATH=/miniconda/bin:$PATH
17+
18+
wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf -
19+
mv go /usr/local/
20+
export PATH=/usr/local/go/bin:$PATH
21+
export GO111MODULE=on
22+
23+
wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz
24+
mv linux-amd64/helm /usr/local/bin/helm

‎.prow/scripts/prepare_integration_test.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
usage()
5+
{
6+
echo "usage: prepare_integration_test.sh [--skip-build true]"
7+
}
8+
9+
while [ "$1" != "" ]; do
10+
case "$1" in
11+
--skip-build ) SKIP_BUILD=true; shift;;
12+
* ) usage; exit 1
13+
esac
14+
shift
15+
done
16+
17+
# Authenticate to Google Cloud and GKE
18+
# ============================================================
19+
GOOGLE_PROJECT_ID=kf-feast
20+
KUBE_CLUSTER_NAME=primary-test-cluster
21+
KUBE_CLUSTER_ZONE=us-central1-a
22+
KEY_FILE=/etc/service-account/service-account.json
23+
24+
gcloud -q auth activate-service-account --key-file=${KEY_FILE}
25+
gcloud -q auth configure-docker
26+
gcloud -q config set project ${GOOGLE_PROJECT_ID}
27+
gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID}
28+
export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE}
29+
30+
# Install Python 3.6, Golang 1.12, Helm and Feast SDK
31+
# ============================================================
32+
. .prow/scripts/install_test_tools.sh
33+
. .prow/scripts/install_feast_sdk.sh
34+
.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME}
35+
36+
# Prepare Feast test data and config
37+
# ============================================================
38+
39+
bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET}
40+
gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH}
41+
42+
BUILD_ID=${BUILD_ID:0:5}
43+
envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml
44+
cd ${FEAST_HOME}/integration-tests/testdata/import_specs
45+
envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
46+
envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
47+
48+
if [[ ! ${SKIP_BUILD} ]]; then
49+
50+
echo "============================================================"
51+
echo "Building Feast for Testing"
52+
echo "============================================================"
53+
cd ${FEAST_HOME}
54+
docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . &
55+
docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . &
56+
wait
57+
docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} &
58+
docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} &
59+
wait
60+
61+
fi
62+
63+
# Switch back context to original directory
64+
set +ex
65+
cd ${FEAST_HOME}

0 commit comments

Comments
 (0)