Skip to content

Commit 5ab6232

Browse files
adchiaadchia
authored
Update release workflow to include new docker images (feast-dev#2108)
* Remove feature transformation server building workflow and merge release workflows. Add/update docker images for lambda feature server, FTS, and java feature server Signed-off-by: Danny Chiao <danny@tecton.ai> * update for ecr Signed-off-by: Danny Chiao <danny@tecton.ai> * Update makefile Signed-off-by: Danny Chiao <danny@tecton.ai> * Separate GCR docker builds into master_only workflow Signed-off-by: Danny Chiao <danny@tecton.ai> * Remove continuous push on merge Signed-off-by: Danny Chiao <danny@tecton.ai> * Update feature server lambda image in constants.py Signed-off-by: Danny Chiao <danny@tecton.ai> * Update docs for lambda Signed-off-by: Danny Chiao <danny@tecton.ai>
1 parent 96393dd commit 5ab6232

7 files changed

Lines changed: 119 additions & 254 deletions

File tree

‎.github/workflows/build_feature_transformation_server.yml‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎.github/workflows/java_release.yml‎

Lines changed: 0 additions & 139 deletions
This file was deleted.

.github/workflows/integration_tests.yml renamed to .github/workflows/master_only.yml

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
name: integration-tests
1+
name: master_only
22

33
on:
44
push:
55
branches:
66
- master
77

88
jobs:
9-
build-docker-image:
9+
build-lambda-docker-image:
1010
runs-on: ubuntu-latest
1111
steps:
1212
- uses: actions/checkout@v2
@@ -46,7 +46,7 @@ jobs:
4646
outputs:
4747
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
4848
integration-test-python:
49-
needs: build-docker-image
49+
needs: build-lambda-docker-image
5050
runs-on: ${{ matrix.os }}
5151
strategy:
5252
fail-fast: false
@@ -113,7 +113,7 @@ jobs:
113113
run: make install-python-ci-dependencies
114114
- name: Test python
115115
env:
116-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
116+
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
117117
FEAST_USAGE: "False"
118118
IS_TEST: "True"
119119
run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
@@ -128,9 +128,46 @@ jobs:
128128
verbose: true
129129
- name: Benchmark python
130130
env:
131-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
131+
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
132132
FEAST_USAGE: "False"
133133
IS_TEST: "True"
134134
run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
135135
- name: Upload Benchmark Artifact to S3
136136
run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks
137+
138+
build-all-docker-images:
139+
runs-on: ubuntu-latest
140+
strategy:
141+
matrix:
142+
component: [ feature-server-aws, feature-server-java, feature-transformation-server ]
143+
env:
144+
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
145+
REGISTRY: gcr.io/kf-feast
146+
steps:
147+
- uses: actions/checkout@v2
148+
- name: Set up QEMU
149+
uses: docker/setup-qemu-action@v1
150+
- name: Set up Docker Buildx
151+
uses: docker/setup-buildx-action@v1
152+
- name: Login to DockerHub
153+
uses: docker/login-action@v1
154+
with:
155+
username: ${{ secrets.DOCKERHUB_USERNAME }}
156+
password: ${{ secrets.DOCKERHUB_TOKEN }}
157+
- name: Set up Cloud SDK
158+
uses: google-github-actions/setup-gcloud@master
159+
with:
160+
project_id: ${{ secrets.GCP_PROJECT_ID }}
161+
service_account_key: ${{ secrets.GCP_SA_KEY }}
162+
export_default_credentials: true
163+
- name: Use gcloud CLI
164+
run: gcloud info
165+
- run: gcloud auth configure-docker --quiet
166+
- name: Get m2 cache
167+
run: |
168+
infra/scripts/download-maven-cache.sh \
169+
--archive-uri ${MAVEN_CACHE} \
170+
--output-dir .
171+
- name: Build image
172+
run: |
173+
make build-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}

‎.github/workflows/release.yml‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,52 +13,53 @@ jobs:
1313
version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
1414
highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
1515
steps:
16-
- uses: actions/checkout@v2
17-
- name: Get release version
18-
id: get_release_version
19-
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
20-
- name: Get release version without prefix
21-
id: get_release_version_without_prefix
22-
env:
23-
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
24-
run: |
25-
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
26-
- name: Get highest semver
27-
id: get_highest_semver
28-
env:
29-
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
30-
run: |
31-
source infra/scripts/setup-common-functions.sh
32-
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
33-
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
34-
echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
35-
fi
36-
- name: Check output
37-
env:
38-
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
39-
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
40-
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
41-
run: |
42-
echo $RELEASE_VERSION
43-
echo $VERSION_WITHOUT_PREFIX
44-
echo $HIGHEST_SEMVER_TAG
16+
- uses: actions/checkout@v2
17+
- name: Get release version
18+
id: get_release_version
19+
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
20+
- name: Get release version without prefix
21+
id: get_release_version_without_prefix
22+
env:
23+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
24+
run: |
25+
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
26+
- name: Get highest semver
27+
id: get_highest_semver
28+
env:
29+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
30+
run: |
31+
source infra/scripts/setup-common-functions.sh
32+
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
33+
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
34+
echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
35+
fi
36+
- name: Check output
37+
env:
38+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
39+
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
40+
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
41+
run: |
42+
echo $RELEASE_VERSION
43+
echo $VERSION_WITHOUT_PREFIX
44+
echo $HIGHEST_SEMVER_TAG
4545
4646
build-publish-docker-images:
47-
runs-on: [ubuntu-latest]
47+
runs-on: ubuntu-latest
4848
needs: get-version
4949
strategy:
5050
matrix:
51-
component: [jupyter]
51+
component: [feature-server-aws, feature-server-java, feature-transformation-server]
5252
env:
5353
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
54+
REGISTRY: feastdev
5455
steps:
5556
- uses: actions/checkout@v2
5657
- name: Set up QEMU
5758
uses: docker/setup-qemu-action@v1
5859
- name: Set up Docker Buildx
5960
uses: docker/setup-buildx-action@v1
6061
- name: Login to DockerHub
61-
uses: docker/login-action@v1
62+
uses: docker/login-action@v1
6263
with:
6364
username: ${{ secrets.DOCKERHUB_USERNAME }}
6465
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -76,27 +77,26 @@ jobs:
7677
infra/scripts/download-maven-cache.sh \
7778
--archive-uri ${MAVEN_CACHE} \
7879
--output-dir .
79-
- name: Build and push versioned images
80+
- name: Build image
81+
run: |
82+
make build-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${RELEASE_VERSION}
83+
env:
84+
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
85+
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
86+
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
87+
- name: Push versioned images
8088
env:
8189
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
8290
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
8391
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
8492
run: |
85-
docker build --build-arg VERSION=$RELEASE_VERSION \
86-
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
87-
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
88-
-t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
89-
-f infra/docker/${{ matrix.component }}/Dockerfile .
90-
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
91-
docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
92-
93+
make push-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${RELEASE_VERSION}
94+
9395
echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
9496
if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
9597
then
96-
docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
97-
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
98-
docker push feastdev/feast-${{ matrix.component }}:latest
99-
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
98+
docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
99+
docker push feastdev/${{ matrix.component }}:latest
100100
fi
101101
102102
publish-helm-charts:
@@ -142,4 +142,7 @@ jobs:
142142
cd sdk/python
143143
python3 -m pip install --user --upgrade setuptools wheel twine
144144
python3 setup.py sdist bdist_wheel
145-
python3 -m twine upload --verbose dist/*
145+
python3 -m twine upload --verbose dist/*
146+
147+
# TODO(adchia): publish java sdk once maven repo is updated
148+
# See https://github.com/feast-dev/feast-java/blob/master/.github/workflows/release.yml#L104

0 commit comments

Comments
 (0)