Skip to content

Commit 4e85104

Browse files
oavdeevoavdeev
committed
Build docker images for e2e
This reverts commit b4f70c3. Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
1 parent 60d78de commit 4e85104

File tree

4 files changed

+68
-11
lines changed

4 files changed

+68
-11
lines changed

‎.github/workflows/complete.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,40 @@ name: complete
33
on: [push, pull_request]
44

55
jobs:
6+
7+
build-push-docker-images-for-e2e-tests:
8+
runs-on: [self-hosted]
9+
strategy:
10+
matrix:
11+
component: [core, serving, jobservice, jupyter, ci]
12+
env:
13+
GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
14+
REGISTRY: gcr.io/kf-feast
15+
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
16+
steps:
17+
- uses: actions/checkout@v2
18+
with:
19+
submodules: recursive
20+
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
21+
with:
22+
version: '290.0.1'
23+
export_default_credentials: true
24+
- run: gcloud auth configure-docker --quiet
25+
- name: Get m2 cache
26+
run: |
27+
infra/scripts/download-maven-cache.sh \
28+
--archive-uri ${MAVEN_CACHE} \
29+
--output-dir $HOME
30+
- name: Build image
31+
run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
32+
- name: Push image
33+
run: |
34+
docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA}
35+
if [ -n "${GITHUB_PR_SHA}" ]; then
36+
docker tag ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_PR_SHA}
37+
docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_PR_SHA}
38+
fi
39+
640
lint-java:
741
container: gcr.io/kf-feast/feast-ci:latest
842
runs-on: [ubuntu-latest]

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Spark ingestion jobs for [Feast](https://github.com/feast-dev/feast)
22

3-
Example usage:
3+
Usage:
44

55
```python
66

‎infra/scripts/azure-runner.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,17 @@ echo "########## Starting e2e tests for ${GIT_REMOTE_URL} ${GIT_TAG} ###########
1616
# Note requires running in root feast directory
1717
source infra/scripts/k8s-common-functions.sh
1818

19+
# Figure out docker image versions
20+
21+
# Those are build from submodule, but tagged with the sha tag of this repo.
22+
export JUPYTER_GIT_TAG=$GIT_TAG
23+
export SERVING_GIT_TAG=$GIT_TAG
24+
export CORE_GIT_TAG=$GIT_TAG
25+
export CI_GIT_TAG=$GIT_TAG
26+
27+
# Jobservice is built by this repo
28+
export JOBSERVICE_GIT_TAG=$GIT_TAG
29+
1930
# Workaround for COPY command in core docker image that pulls local maven repo into the image
2031
# itself.
2132
mkdir .m2 2>/dev/null || true
@@ -36,9 +47,12 @@ RELEASE=sparkop
3647
# Delete old helm release and PVCs
3748
k8s_cleanup "$RELEASE" "$NAMESPACE"
3849

39-
wait_for_images "${DOCKER_REPOSITORY}" "${GIT_TAG}"
50+
# Wait for CI and jobservice image to be built
51+
wait_for_image "${DOCKER_REPOSITORY}" feast-ci "${CI_GIT_TAG}"
52+
wait_for_image "${DOCKER_REPOSITORY}" feast-jobservice "${JOBSERVICE_GIT_TAG}"
4053

41-
# Helm install everything in a namespace
54+
# Helm install everything in a namespace. Note that this function will use XXX_GIT_TAG variables
55+
# we've set above to find the image versions.
4256
helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \
4357
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
4458
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}"

‎infra/scripts/k8s-common-functions.sh‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22

33
set -euo pipefail
44

5-
function wait_for_images {
5+
function wait_for_image {
66
local DOCKER_REPOSITORY=$1
7-
local GIT_TAG=$2
7+
local IMAGE_NAME=$2
8+
local GIT_TAG=$3
89
# Wait for images to be available in the docker repository; ci is the last image built
9-
timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
10+
echo "Waiting for ${DOCKER_REPOSITORY}/${IMAGE_NAME}:${GIT_TAG} to become available"
11+
timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/${IMAGE_NAME} --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
1012
}
1113

1214
function k8s_cleanup {
@@ -42,8 +44,15 @@ function helm_install {
4244
# Args:
4345
# $RELEASE is helm release name
4446
# $DOCKER_REPOSITORY is the docker repo containing feast images tagged with $GIT_TAG
45-
# ... you can pass additional args to this function that are passed on to helm install
4647
# $NAMESPACE is the namespace name
48+
# $GIT_TAG is the git tag to use when pulling the images. This can also be overridden
49+
# on per-image basis, that is, if these are set, they'll be used instead:
50+
# $JUPYTER_GIT_TAG
51+
# $SERVING_GIT_TAG
52+
# $CORE_GIT_TAG
53+
# $JOBSERVICE_GIT_TAG
54+
#
55+
# ... you can pass additional args to this function that are passed on to helm install
4756

4857
local RELEASE=$1
4958
local DOCKER_REPOSITORY=$2
@@ -59,13 +68,13 @@ function helm_install {
5968
if ! time helm install --wait "$RELEASE" "${HELM_CHART_LOCATION:-./infra/charts/feast}" \
6069
--timeout 15m \
6170
--set "feast-jupyter.image.repository=${DOCKER_REPOSITORY}/feast-jupyter" \
62-
--set "feast-jupyter.image.tag=${GIT_TAG}" \
71+
--set "feast-jupyter.image.tag=${JUPYTER_GIT_TAG:-$GIT_TAG}" \
6372
--set "feast-online-serving.image.repository=${DOCKER_REPOSITORY}/feast-serving" \
64-
--set "feast-online-serving.image.tag=${GIT_TAG}" \
73+
--set "feast-online-serving.image.tag=${SERVING_GIT_TAG:-$GIT_TAG}" \
6574
--set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
66-
--set "feast-jobservice.image.tag=${GIT_TAG}" \
75+
--set "feast-jobservice.image.tag=${JOBSERVICE_GIT_TAG:-$GIT_TAG}" \
6776
--set "feast-core.image.repository=${DOCKER_REPOSITORY}/feast-core" \
68-
--set "feast-core.image.tag=${GIT_TAG}" \
77+
--set "feast-core.image.tag=${CORE_GIT_TAG:-$GIT_TAG}" \
6978
--set "prometheus-statsd-exporter.enabled=false" \
7079
--set "prometheus.enabled=false" \
7180
--set "grafana.enabled=false" \

0 commit comments

Comments
 (0)