Skip to content

chore: Remove lambda feature server image from ci #4273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
franciscojavierarceo merged 1 commit into from
Jun 13, 2024
Merged

chore: Remove lambda feature server image from ci #4273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 0 additions & 60 deletions .github/fork_workflows/fork_pr_integration_tests_aws.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,64 +3,6 @@ name: fork-pr-integration-tests-aws
on: [pull_request]

jobs:
build-docker-image:
if: github.repository == 'your github repo' # swap here with your project id
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# pull_request_target runs the workflow in the context of the base repo
# as such actions/checkout needs to be explicit configured to retrieve
# code from the PR.
ref: refs/pull/${{ github.event.pull_request.number }}/merge
submodules: recursive
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Set up AWS SDK
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Set ECR image tag
id: image-tag
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
- name: Cache Public ECR Image
id: lambda_python_3_11
uses: actions/cache@v2
with:
path: ~/cache
key: lambda_python_3_11
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
if: steps.cache-primes.outputs.cache-hit != 'true'
run: |
mkdir -p ~/cache
docker pull public.ecr.aws/lambda/python:3.11
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
- name: Handle Cache Hit (load docker image from tar file)
if: steps.cache-primes.outputs.cache-hit == 'true'
run: |
docker load -i ~/cache/lambda_python_3_11.tar
- name: Build and push
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: feast-python-server
run: |
docker build \
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
--load \
.
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
outputs:
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
integration-test-python:
if: github.repository == 'your github repo' # swap here with your project id
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -138,8 +80,6 @@ jobs:
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
- name: Test python
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
env:
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
run: |
pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "aws and not Snowflake and not BigQuery and not minio_registry"
pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "File and not Snowflake and not BigQuery and not minio_registry"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build_wheels.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ jobs:
needs: get-version
strategy:
matrix:
component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server]
component: [feature-server, feature-server-java, feature-transformation-server]
env:
REGISTRY: feastdev
steps:
Expand Down
55 changes: 0 additions & 55 deletions .github/workflows/master_only.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,61 +6,8 @@ on:
- master

jobs:
build-lambda-docker-image:
if: github.repository == 'feast-dev/feast'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Set up AWS SDK
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Set ECR image tag
id: image-tag
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
- name: Cache Public ECR Image
id: lambda_python_3_11
uses: actions/cache@v2
with:
path: ~/cache
key: lambda_python_3_11
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
if: steps.cache-primes.outputs.cache-hit != 'true'
run: |
mkdir -p ~/cache
docker pull public.ecr.aws/lambda/python:3.11
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
- name: Handle Cache Hit (load docker image from tar file)
if: steps.cache-primes.outputs.cache-hit == 'true'
run: |
docker load -i ~/cache/lambda_python_3_11.tar
- name: Build and push
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: feast-python-server
run: |
docker build \
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
--load \
.
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
outputs:
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
integration-test-python:
if: github.repository == 'feast-dev/feast'
needs: build-lambda-docker-image
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -125,7 +72,6 @@ jobs:
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
- name: Test python and go
env:
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
Expand All @@ -134,7 +80,6 @@ jobs:
run: make test-python-integration
- name: Benchmark python
env:
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
Expand Down
59 changes: 1 addition & 58 deletions .github/workflows/nightly-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,65 +61,9 @@ jobs:
run: gcloud info
- name: Run DynamoDB / Bigtable cleanup script
run: python infra/scripts/cleanup_ci.py
build-docker-image:
if: github.repository == 'feast-dev/feast'
needs: [check_date]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: master
submodules: recursive
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Set up AWS SDK
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Set ECR image tag
id: image-tag
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
- name: Cache Public ECR Image
id: lambda_python_3_11
uses: actions/cache@v4
with:
path: ~/cache
key: lambda_python_3_11
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
if: steps.lambda_python_3_11.outputs.cache-hit != 'true'
run: |
mkdir -p ~/cache
docker pull public.ecr.aws/lambda/python:3.11
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
- name: Handle Cache Hit (load docker image from tar file)
if: steps.lambda_python_3_11.outputs.cache-hit == 'true'
run: |
docker load -i ~/cache/lambda_python_3_11.tar
- name: Build and push
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: feast-python-server
run: |
docker build \
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
--load \
.
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
outputs:
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
integration-test-python:
if: github.repository == 'feast-dev/feast'
needs: [check_date, build-docker-image, cleanup_dynamo_tables]
needs: [check_date, cleanup_dynamo_tables]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -205,7 +149,6 @@ jobs:
- name: Test python
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
env:
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
Expand Down
63 changes: 0 additions & 63 deletions .github/workflows/pr_integration_tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,75 +13,12 @@ on:
# cancel-in-progress: true

jobs:
build-docker-image:
# when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
if:
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
github.repository == 'feast-dev/feast'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# pull_request_target runs the workflow in the context of the base repo
# as such actions/checkout needs to be explicit configured to retrieve
# code from the PR.
ref: refs/pull/${{ github.event.pull_request.number }}/merge
submodules: recursive
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Set up AWS SDK
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Set ECR image tag
id: image-tag
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
- name: Cache Public ECR Image
id: lambda_python_3_11
uses: actions/cache@v2
with:
path: ~/cache
key: lambda_python_3_11
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
if: steps.cache-primes.outputs.cache-hit != 'true'
run: |
mkdir -p ~/cache
docker pull public.ecr.aws/lambda/python:3.11
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
- name: Handle Cache Hit (load docker image from tar file)
if: steps.cache-primes.outputs.cache-hit == 'true'
run: |
docker load -i ~/cache/lambda_python_3_11.tar
- name: Build and push
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: feast-python-server
run: |
docker build \
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
--load \
.
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
outputs:
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
integration-test-python:
# when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
if:
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
github.repository == 'feast-dev/feast'
needs: build-docker-image
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
needs: [get-version, publish-python-sdk]
strategy:
matrix:
component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator]
component: [feature-server, feature-server-java, feature-transformation-server, feast-operator]
env:
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
REGISTRY: feastdev
Expand Down