diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index aa89ece1776..6eb8b8feff0 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -3,64 +3,6 @@ name: fork-pr-integration-tests-aws on: [pull_request] jobs: - build-docker-image: - if: github.repository == 'your github repo' # swap here with your project id - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'your github repo' # swap here with your project id runs-on: ${{ matrix.os }} @@ -138,8 +80,6 @@ jobs: docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak - env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} run: | pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "aws and not Snowflake and not BigQuery and not minio_registry" pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "File and not Snowflake and not BigQuery and not minio_registry" diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 596eef2b52c..f04015a9892 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -116,7 +116,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] + component: [feature-server, feature-server-java, feature-transformation-server] env: REGISTRY: feastdev steps: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 1d6850e4d8e..1b401997a7b 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -6,61 +6,8 @@ on: - master jobs: - build-lambda-docker-image: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'feast-dev/feast' - needs: build-lambda-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -125,7 +72,6 @@ jobs: docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python and go env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} @@ -134,7 +80,6 @@ jobs: run: make test-python-integration - name: Benchmark python env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 8a6ed2d7a73..11c91af2d7b 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -61,65 +61,9 @@ jobs: run: gcloud info - name: Run DynamoDB / Bigtable cleanup script run: python infra/scripts/cleanup_ci.py - build-docker-image: - if: github.repository == 'feast-dev/feast' - needs: [check_date] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: master - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v4 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.lambda_python_3_11.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.lambda_python_3_11.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'feast-dev/feast' - needs: [check_date, build-docker-image, cleanup_dynamo_tables] + needs: [check_date, cleanup_dynamo_tables] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -205,7 +149,6 @@ jobs: - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 3081d418fcf..f4a9132d292 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -13,75 +13,12 @@ on: # cancel-in-progress: true jobs: - build-docker-image: - # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - if: - ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || - (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && - github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && github.repository == 'feast-dev/feast' - needs: build-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 914e5a233c7..e56296ec4b4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: [get-version, publish-python-sdk] strategy: matrix: - component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator] + component: [feature-server, feature-server-java, feature-transformation-server, feast-operator] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev