Skip to content

Commit 68fc3b7

Browse files
davidheryantofeast-ci-bot
davidheryanto
authored and
feast-ci-bot
committed
Add Prow jobs to automate the release of Docker images and Python SDK (feast-dev#369)
* Add CI script for push to master and release tag. - Publish Python SDK to pypi - Publish Docker images - Publish Helm chart Also update Prow config for test-infra with image tag: v20191211 https://github.com/kubernetes/test-infra/tree/821a860177455dab60160c7624950c2da5c092ed/prow/cluster * Install Python SDK with editable mode during testing So that setuptools_scm does not complain about missing version * Use editable mode for Python SDK installation during testing * Use consistent separator, dash instead of underscore, in script names and options * Remove duplicate README.md in sdk/python folder Reuse the same README in repository root * Push docker images with commit SHA as the tag as well, upon push to master branch
1 parent c1e4c57 commit 68fc3b7

11 files changed

Lines changed: 297 additions & 40 deletions

‎.prow/config.yaml‎

Lines changed: 170 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,41 @@ prowjob_namespace: default
22
pod_namespace: test-pods
33

44
plank:
5-
allow_cancellations: true
6-
job_url_prefix: http://prow.feast.ai/view/gcs
5+
job_url_prefix_config:
6+
"*": http://prow.feast.ai/view/gcs
77
report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})'
8-
default_decoration_config:
9-
timeout: 7200000000000 # 2h
10-
grace_period: 15000000000 # 15s
11-
utility_images:
12-
clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
13-
initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
14-
entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
15-
sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
16-
gcs_configuration:
17-
bucket: feast-templocation-kf-feast
18-
path_strategy: explicit
19-
gcs_credentials_secret: prow-service-account
8+
pod_pending_timeout: 60m
9+
default_decoration_configs:
10+
"*":
11+
timeout: 1h
12+
grace_period: 15s
13+
utility_images:
14+
clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
15+
initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
16+
entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
17+
sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
18+
gcs_configuration:
19+
bucket: feast-templocation-kf-feast
20+
path_strategy: explicit
21+
gcs_credentials_secret: prow-service-account
2022

2123
deck:
2224
tide_update_period: 1s
2325
spyglass:
24-
size_limit: 50e+6 # 50MB
25-
viewers:
26-
"started.json|finished.json": ["metadata"]
27-
"build-log.txt": ["buildlog"]
28-
"report.xml": ["junit"]
29-
"artifacts/.*\\.xml": ["junit"]
30-
"surefire-reports/.*\\.xml": ["junit"]
26+
size_limit: 10e+6 # 10MB
27+
lenses:
28+
- lens:
29+
name: metadata
30+
required_files:
31+
- started.json|finished.json
32+
- lens:
33+
name: buildlog
34+
required_files:
35+
- build-log.txt
36+
- lens:
37+
name: junit
38+
required_files:
39+
- artifacts/.*\.xml
3140

3241
tide:
3342
queries:
@@ -47,7 +56,9 @@ tide:
4756
blocker_label: merge-blocker
4857
squash_label: tide/squash
4958

50-
# presubmits list Prow jobs that run on pull requests
59+
# presubmits and postsubmits configure ProwJobs:
60+
# https://github.com/kubernetes/test-infra/blob/6571843b1aa7bd6cf577a7a8b9e9971241f424d5/prow/jobs.md
61+
5162
presubmits:
5263
gojek/feast:
5364
- name: test-core-and-ingestion
@@ -131,8 +142,140 @@ presubmits:
131142
- name: service-account
132143
mountPath: "/etc/service-account"
133144

134-
# TODO: do a release when a git tag is pushed
135-
#
136-
# postsubmits list Prow jobs that run on every push
137-
# postsubmits:
138-
# gojek/feast:
145+
postsubmits:
146+
gojek/feast:
147+
- name: publish-python-sdk
148+
decorate: true
149+
spec:
150+
containers:
151+
- image: python:3
152+
command:
153+
- sh
154+
- -c
155+
- |
156+
.prow/scripts/publish-python-sdk.sh \
157+
--directory-path sdk/python --repository pypi
158+
volumeMounts:
159+
- name: pypirc
160+
mountPath: /root/.pypirc
161+
subPath: .pypirc
162+
readOnly: true
163+
volumes:
164+
- name: pypirc
165+
secret:
166+
secretName: pypirc
167+
branches:
168+
# Filter on tags with semantic versioning, prefixed with "v"
169+
# https://github.com/semver/semver/issues/232
170+
- ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
171+
172+
- name: publish-docker-images
173+
decorate: true
174+
spec:
175+
containers:
176+
- image: google/cloud-sdk:273.0.0
177+
command:
178+
- bash
179+
- -c
180+
- |
181+
.prow/scripts/download-maven-cache.sh \
182+
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
183+
--output-dir $PWD/
184+
185+
if [ $PULL_BASE_REF == "master" ]; then
186+
187+
.prow/scripts/publish-docker-image.sh \
188+
--repository gcr.io/kf-feast/feast-core \
189+
--tag dev \
190+
--file infra/docker/core/Dockerfile \
191+
--google-service-account-file /etc/gcloud/service-account.json
192+
193+
.prow/scripts/publish-docker-image.sh \
194+
--repository gcr.io/kf-feast/feast-serving \
195+
--tag dev \
196+
--file infra/docker/serving/Dockerfile \
197+
--google-service-account-file /etc/gcloud/service-account.json
198+
199+
docker tag gcr.io/kf-feast/feast-core:dev gcr.io/kf-feast/feast-core:${PULL_BASE_SHA}
200+
docker push gcr.io/kf-feast/feast-core:${PULL_BASE_SHA}
201+
202+
docker tag gcr.io/kf-feast/feast-serving:dev gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA}
203+
docker push gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA}
204+
205+
else
206+
207+
.prow/scripts/publish-docker-image.sh \
208+
--repository gcr.io/kf-feast/feast-core \
209+
--tag ${PULL_BASE_REF:1} \
210+
--file infra/docker/core/Dockerfile \
211+
--google-service-account-file /etc/gcloud/service-account.json
212+
213+
.prow/scripts/publish-docker-image.sh \
214+
--repository gcr.io/kf-feast/feast-serving \
215+
--tag ${PULL_BASE_REF:1} \
216+
--file infra/docker/serving/Dockerfile \
217+
--google-service-account-file /etc/gcloud/service-account.json
218+
219+
docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-core:latest
220+
docker push gcr.io/kf-feast/feast-core:latest
221+
222+
docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-serving:latest
223+
docker push gcr.io/kf-feast/feast-serving:latest
224+
225+
fi
226+
volumeMounts:
227+
- name: docker-socket
228+
mountPath: /var/run/docker.sock
229+
- name: service-account
230+
mountPath: /etc/gcloud/service-account.json
231+
subPath: service-account.json
232+
readOnly: true
233+
securityContext:
234+
privileged: true
235+
volumes:
236+
- name: docker-socket
237+
hostPath:
238+
path: /var/run/docker.sock
239+
- name: service-account
240+
secret:
241+
secretName: feast-service-account
242+
branches:
243+
- ^master$
244+
- ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
245+
246+
- name: publish-helm-chart
247+
decorate: true
248+
spec:
249+
containers:
250+
- image: google/cloud-sdk:273.0.0-slim
251+
command:
252+
- bash
253+
- -c
254+
- |
255+
gcloud auth activate-service-account --key-file /etc/gcloud/service-account.json
256+
257+
curl -s https://get.helm.sh/helm-v2.16.1-linux-amd64.tar.gz | tar -C /tmp -xz
258+
mv /tmp/linux-amd64/helm /usr/bin/helm
259+
helm init --client-only
260+
261+
sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/Chart.yaml
262+
sed -i "/ version: /c\ version: ${PULL_BASE_REF:1}" infra/charts/feast/requirements.yaml
263+
264+
sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/Chart.yaml
265+
sed -i "/ tag: /c\ tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/values.yaml
266+
267+
sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/Chart.yaml
268+
sed -i "/ tag: /c\ tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/values.yaml
269+
270+
.prow/scripts/sync-helm-charts.sh
271+
volumeMounts:
272+
- name: service-account
273+
mountPath: /etc/gcloud/service-account.json
274+
subPath: service-account.json
275+
readOnly: true
276+
volumes:
277+
- name: service-account
278+
secret:
279+
secretName: feast-service-account
280+
branches:
281+
- ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$

‎.prow/scripts/download-maven-cache.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi
2626
# Install Google Cloud SDK if gsutil command not exists
2727
if [[ ! $(command -v gsutil) ]]; then
2828
CURRENT_DIR=$(dirname "$BASH_SOURCE")
29-
. "${CURRENT_DIR}"/install_google_cloud_sdk.sh
29+
. "${CURRENT_DIR}"/install-google-cloud-sdk.sh
3030
fi
3131

3232
gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar

.prow/scripts/install_google_cloud_sdk.sh renamed to .prow/scripts/install-google-cloud-sdk.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ set -e
33

44
usage()
55
{
6-
echo "usage: . install_google_cloud_sdk.sh
6+
echo "usage: . install-google-cloud-sdk.sh
77
[--with-key-file local file path to service account json]
88
9-
NOTE: requires 'dot' before install_google_cloud_sdk.sh
9+
NOTE: requires 'dot' before install-google-cloud-sdk.sh
1010
so that the PATH variable is exported succesfully to
1111
the calling process, i.e. you don't need to provide
1212
full path to gcloud command after installation

‎.prow/scripts/publish-docker-image.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
set -o pipefail
5+
6+
usage()
7+
{
8+
echo "usage: publish-docker-image.sh
9+
10+
--repository the target repository to upload the Docker image, example:
11+
gcr.io/kf-feast/feast-core
12+
13+
--tag the tag for the Docker image, example: 1.0.4
14+
15+
--file path to the Dockerfile
16+
17+
[--google-service-account-file
18+
path to Google Cloud service account JSON key file]
19+
"
20+
}
21+
22+
while [ "$1" != "" ]; do
23+
case "$1" in
24+
--repository ) REPOSITORY="$2"; shift;;
25+
--tag ) TAG="$2"; shift;;
26+
--file ) FILE="$2"; shift;;
27+
--google-service-account-file ) GOOGLE_SERVICE_ACCOUNT_FILE="$2"; shift;;
28+
-h | --help ) usage; exit;;
29+
* ) usage; exit 1
30+
esac
31+
shift
32+
done
33+
34+
if [ -z $REPOSITORY ]; then usage; exit 1; fi
35+
if [ -z $TAG ]; then usage; exit 1; fi
36+
if [ -z $FILE ]; then usage; exit 1; fi
37+
38+
if [ $GOOGLE_SERVICE_ACCOUNT_FILE ]; then
39+
gcloud -q auth activate-service-account --key-file $GOOGLE_SERVICE_ACCOUNT_FILE
40+
gcloud -q auth configure-docker
41+
fi
42+
43+
echo "============================================================"
44+
echo "Building Docker image $REPOSITORY:$TAG"
45+
echo "============================================================"
46+
docker build -t $REPOSITORY:$TAG --build-arg REVISION=$TAG -f $FILE .
47+
48+
echo "============================================================"
49+
echo "Pushing Docker image $REPOSITORY:$TAG"
50+
echo "============================================================"
51+
docker push $REPOSITORY:$TAG

‎.prow/scripts/publish-python-sdk.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
set -o pipefail
5+
6+
usage()
7+
{
8+
echo "usage: publish-python-sdk.sh
9+
10+
--directory-path absolute path to the python package, this directory
11+
should contain 'setup.py' file
12+
13+
--repository the repository name where the package will be uploaded,
14+
check your .pypirc configuration file for the list of
15+
valid repositories, usually it's 'pypi' or 'testpypi'
16+
"
17+
}
18+
19+
while [ "$1" != "" ]; do
20+
case "$1" in
21+
--directory-path ) DIRECTORY_PATH="$2"; shift;;
22+
--repository ) REPOSITORY="$2"; shift;;
23+
-h | --help ) usage; exit;;
24+
* ) usage; exit 1
25+
esac
26+
shift
27+
done
28+
29+
if [ -z $DIRECTORY_PATH ]; then usage; exit 1; fi
30+
if [ -z $REPOSITORY ]; then usage; exit 1; fi
31+
32+
ORIGINAL_DIR=$PWD
33+
cd $DIRECTORY_PATH
34+
35+
echo "============================================================"
36+
echo "Generating distribution archives"
37+
echo "============================================================"
38+
python3 -m pip install --user --upgrade setuptools wheel
39+
python3 setup.py sdist bdist_wheel
40+
41+
echo "============================================================"
42+
echo "Uploading distribution archives"
43+
echo "============================================================"
44+
python3 -m pip install --user --upgrade twine
45+
python3 -m twine upload --repository $REPOSITORY dist/*
46+
47+
cd $ORIGINAL_DIR

‎.prow/scripts/sync-helm-charts.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@ if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$sync_dir"
4646
gsutil -m rsync "$sync_dir" "$bucket"
4747

4848
# Make sure index.yaml is synced last
49-
gsutil cp "$index_dir/index.yaml" "$bucket"
49+
gsutil -h "Cache-Control:no-cache,max-age=0" cp "$index_dir/index.yaml" "$bucket"
5050
else
5151
log_error "Exiting because unable to update index. Not safe to push update."
5252
exit 1
5353
fi
5454

5555
ls -l "$sync_dir"
5656

57-
exit "$exit_code"
57+
exit "$exit_code"

‎.prow/scripts/test-end-to-end-batch.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ Installing gcloud SDK
2828
"
2929
if [[ ! $(command -v gsutil) ]]; then
3030
CURRENT_DIR=$(dirname "$BASH_SOURCE")
31-
. "${CURRENT_DIR}"/install_google_cloud_sdk.sh
31+
. "${CURRENT_DIR}"/install-google-cloud-sdk.sh
3232
fi
3333

3434
export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json
@@ -210,7 +210,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
210210
source ~/.bashrc
211211

212212
# Install Feast Python SDK and test requirements
213-
pip install -q sdk/python
213+
pip install -qe sdk/python
214214
pip install -qr tests/e2e/requirements.txt
215215

216216
echo "

‎.prow/scripts/test-end-to-end.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
191191
source ~/.bashrc
192192

193193
# Install Feast Python SDK and test requirements
194-
pip install -q sdk/python
194+
pip install -qe sdk/python
195195
pip install -qr tests/e2e/requirements.txt
196196

197197
echo "

‎.prow/scripts/test-python-sdk.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ LOGS_ARTIFACT_PATH=/logs/artifacts
77

88
cd sdk/python
99
pip install -r requirements-ci.txt
10-
pip install .
10+
pip install -e .
1111
pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml

0 commit comments

Comments
 (0)