Skip to content

Commit f2079fb

Browse files
lokeshranginenilokeshrangineni
committed
squashing last 61 commits.
Merge remote-tracking branch 'fork/feature/adding-remote-onlinestore-rebase' into feature/adding-remote-onlinestore-rebase Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com>
1 parent 1f17b9b commit f2079fb

File tree

132 files changed

+5018
-2327
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

132 files changed

+5018
-2327
lines changed

‎.github/fork_workflows/fork_pr_integration_tests_aws.yml‎

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -3,64 +3,6 @@ name: fork-pr-integration-tests-aws
33
on: [pull_request]
44

55
jobs:
6-
build-docker-image:
7-
if: github.repository == 'your github repo' # swap here with your project id
8-
runs-on: ubuntu-latest
9-
steps:
10-
- uses: actions/checkout@v4
11-
with:
12-
# pull_request_target runs the workflow in the context of the base repo
13-
# as such actions/checkout needs to be explicit configured to retrieve
14-
# code from the PR.
15-
ref: refs/pull/${{ github.event.pull_request.number }}/merge
16-
submodules: recursive
17-
- name: Set up QEMU
18-
uses: docker/setup-qemu-action@v1
19-
- name: Set up Docker Buildx
20-
uses: docker/setup-buildx-action@v2
21-
with:
22-
install: true
23-
- name: Set up AWS SDK
24-
uses: aws-actions/configure-aws-credentials@v1
25-
with:
26-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
27-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
28-
aws-region: us-west-2
29-
- name: Login to Amazon ECR
30-
id: login-ecr
31-
uses: aws-actions/amazon-ecr-login@v1
32-
- name: Set ECR image tag
33-
id: image-tag
34-
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
35-
- name: Cache Public ECR Image
36-
id: lambda_python_3_11
37-
uses: actions/cache@v2
38-
with:
39-
path: ~/cache
40-
key: lambda_python_3_11
41-
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
42-
if: steps.cache-primes.outputs.cache-hit != 'true'
43-
run: |
44-
mkdir -p ~/cache
45-
docker pull public.ecr.aws/lambda/python:3.11
46-
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
47-
- name: Handle Cache Hit (load docker image from tar file)
48-
if: steps.cache-primes.outputs.cache-hit == 'true'
49-
run: |
50-
docker load -i ~/cache/lambda_python_3_11.tar
51-
- name: Build and push
52-
env:
53-
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
54-
ECR_REPOSITORY: feast-python-server
55-
run: |
56-
docker build \
57-
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
58-
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
59-
--load \
60-
.
61-
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
62-
outputs:
63-
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
646
integration-test-python:
657
if: github.repository == 'your github repo' # swap here with your project id
668
runs-on: ${{ matrix.os }}
@@ -138,8 +80,6 @@ jobs:
13880
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
13981
- name: Test python
14082
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
141-
env:
142-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
14383
run: |
14484
pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "aws and not Snowflake and not BigQuery and not minio_registry"
14585
pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "File and not Snowflake and not BigQuery and not minio_registry"

‎.github/workflows/build_wheels.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ jobs:
116116
needs: get-version
117117
strategy:
118118
matrix:
119-
component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server]
119+
component: [feature-server, feature-server-java, feature-transformation-server]
120120
env:
121121
REGISTRY: feastdev
122122
steps:

‎.github/workflows/master_only.yml‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,61 +6,8 @@ on:
66
- master
77

88
jobs:
9-
build-lambda-docker-image:
10-
if: github.repository == 'feast-dev/feast'
11-
runs-on: ubuntu-latest
12-
steps:
13-
- uses: actions/checkout@v4
14-
- name: Set up QEMU
15-
uses: docker/setup-qemu-action@v1
16-
- name: Set up Docker Buildx
17-
uses: docker/setup-buildx-action@v2
18-
with:
19-
install: true
20-
- name: Set up AWS SDK
21-
uses: aws-actions/configure-aws-credentials@v1
22-
with:
23-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
24-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
25-
aws-region: us-west-2
26-
- name: Login to Amazon ECR
27-
id: login-ecr
28-
uses: aws-actions/amazon-ecr-login@v1
29-
- name: Set ECR image tag
30-
id: image-tag
31-
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
32-
- name: Cache Public ECR Image
33-
id: lambda_python_3_11
34-
uses: actions/cache@v2
35-
with:
36-
path: ~/cache
37-
key: lambda_python_3_11
38-
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
39-
if: steps.cache-primes.outputs.cache-hit != 'true'
40-
run: |
41-
mkdir -p ~/cache
42-
docker pull public.ecr.aws/lambda/python:3.11
43-
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
44-
- name: Handle Cache Hit (load docker image from tar file)
45-
if: steps.cache-primes.outputs.cache-hit == 'true'
46-
run: |
47-
docker load -i ~/cache/lambda_python_3_11.tar
48-
- name: Build and push
49-
env:
50-
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
51-
ECR_REPOSITORY: feast-python-server
52-
run: |
53-
docker build \
54-
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
55-
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
56-
--load \
57-
.
58-
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
59-
outputs:
60-
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
619
integration-test-python:
6210
if: github.repository == 'feast-dev/feast'
63-
needs: build-lambda-docker-image
6411
runs-on: ${{ matrix.os }}
6512
strategy:
6613
fail-fast: false
@@ -125,7 +72,6 @@ jobs:
12572
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
12673
- name: Test python and go
12774
env:
128-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
12975
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
13076
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
13177
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
@@ -134,7 +80,6 @@ jobs:
13480
run: make test-python-integration
13581
- name: Benchmark python
13682
env:
137-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
13883
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
13984
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
14085
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}

‎.github/workflows/nightly-ci.yml‎

Lines changed: 1 addition & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -61,65 +61,9 @@ jobs:
6161
run: gcloud info
6262
- name: Run DynamoDB / Bigtable cleanup script
6363
run: python infra/scripts/cleanup_ci.py
64-
build-docker-image:
65-
if: github.repository == 'feast-dev/feast'
66-
needs: [check_date]
67-
runs-on: ubuntu-latest
68-
steps:
69-
- uses: actions/checkout@v4
70-
with:
71-
ref: master
72-
submodules: recursive
73-
- name: Set up QEMU
74-
uses: docker/setup-qemu-action@v1
75-
- name: Set up Docker Buildx
76-
uses: docker/setup-buildx-action@v2
77-
with:
78-
install: true
79-
- name: Set up AWS SDK
80-
uses: aws-actions/configure-aws-credentials@v1
81-
with:
82-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
83-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
84-
aws-region: us-west-2
85-
- name: Login to Amazon ECR
86-
id: login-ecr
87-
uses: aws-actions/amazon-ecr-login@v1
88-
- name: Set ECR image tag
89-
id: image-tag
90-
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
91-
- name: Cache Public ECR Image
92-
id: lambda_python_3_11
93-
uses: actions/cache@v4
94-
with:
95-
path: ~/cache
96-
key: lambda_python_3_11
97-
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
98-
if: steps.lambda_python_3_11.outputs.cache-hit != 'true'
99-
run: |
100-
mkdir -p ~/cache
101-
docker pull public.ecr.aws/lambda/python:3.11
102-
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
103-
- name: Handle Cache Hit (load docker image from tar file)
104-
if: steps.lambda_python_3_11.outputs.cache-hit == 'true'
105-
run: |
106-
docker load -i ~/cache/lambda_python_3_11.tar
107-
- name: Build and push
108-
env:
109-
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
110-
ECR_REPOSITORY: feast-python-server
111-
run: |
112-
docker build \
113-
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
114-
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
115-
--load \
116-
.
117-
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
118-
outputs:
119-
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
12064
integration-test-python:
12165
if: github.repository == 'feast-dev/feast'
122-
needs: [check_date, build-docker-image, cleanup_dynamo_tables]
66+
needs: [check_date, cleanup_dynamo_tables]
12367
runs-on: ${{ matrix.os }}
12468
strategy:
12569
fail-fast: false
@@ -205,7 +149,6 @@ jobs:
205149
- name: Test python
206150
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
207151
env:
208-
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
209152
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
210153
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
211154
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 0 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -13,75 +13,12 @@ on:
1313
# cancel-in-progress: true
1414

1515
jobs:
16-
build-docker-image:
17-
# when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
18-
if:
19-
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
20-
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
21-
github.repository == 'feast-dev/feast'
22-
runs-on: ubuntu-latest
23-
steps:
24-
- uses: actions/checkout@v4
25-
with:
26-
# pull_request_target runs the workflow in the context of the base repo
27-
# as such actions/checkout needs to be explicit configured to retrieve
28-
# code from the PR.
29-
ref: refs/pull/${{ github.event.pull_request.number }}/merge
30-
submodules: recursive
31-
- name: Set up QEMU
32-
uses: docker/setup-qemu-action@v1
33-
- name: Set up Docker Buildx
34-
uses: docker/setup-buildx-action@v2
35-
with:
36-
install: true
37-
- name: Set up AWS SDK
38-
uses: aws-actions/configure-aws-credentials@v1
39-
with:
40-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
41-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
42-
aws-region: us-west-2
43-
- name: Login to Amazon ECR
44-
id: login-ecr
45-
uses: aws-actions/amazon-ecr-login@v1
46-
- name: Set ECR image tag
47-
id: image-tag
48-
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
49-
- name: Cache Public ECR Image
50-
id: lambda_python_3_11
51-
uses: actions/cache@v2
52-
with:
53-
path: ~/cache
54-
key: lambda_python_3_11
55-
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
56-
if: steps.cache-primes.outputs.cache-hit != 'true'
57-
run: |
58-
mkdir -p ~/cache
59-
docker pull public.ecr.aws/lambda/python:3.11
60-
docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
61-
- name: Handle Cache Hit (load docker image from tar file)
62-
if: steps.cache-primes.outputs.cache-hit == 'true'
63-
run: |
64-
docker load -i ~/cache/lambda_python_3_11.tar
65-
- name: Build and push
66-
env:
67-
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
68-
ECR_REPOSITORY: feast-python-server
69-
run: |
70-
docker build \
71-
--file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
72-
--tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
73-
--load \
74-
.
75-
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
76-
outputs:
77-
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
7816
integration-test-python:
7917
# when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
8018
if:
8119
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
8220
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
8321
github.repository == 'feast-dev/feast'
84-
needs: build-docker-image
8522
runs-on: ${{ matrix.os }}
8623
strategy:
8724
fail-fast: false

‎.github/workflows/publish.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
needs: [get-version, publish-python-sdk]
5050
strategy:
5151
matrix:
52-
component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator]
52+
component: [feature-server, feature-server-java, feature-transformation-server, feast-operator]
5353
env:
5454
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
5555
REGISTRY: feastdev

‎.gitpod.Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM gitpod/workspace-base
2+
RUN sudo apt-get update && sudo apt-get install -y python3-dev python3-setuptools python3-pip python-is-python3 && sudo rm -rf /var/lib/apt/lists/*
3+
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
4+
RUN curl -fsSL https://pixi.sh/install.sh | bash
5+
ENV PATH=$PATH:/home/gitpod/.cargo/bin
6+
RUN curl -s "https://get.sdkman.io" | bash
7+
SHELL ["/bin/bash", "-c"]
8+
RUN source "/home/gitpod/.sdkman/bin/sdkman-init.sh" && sdk install java 14.0.2-zulu

‎.gitpod.yml‎

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,20 @@
11
# https://www.gitpod.io/docs/config-gitpod-file
2+
image:
3+
file: .gitpod.Dockerfile
4+
25
tasks:
36
- init: |
4-
python -m venv venv
5-
source venv/bin/activate
6-
7-
pip install pre-commit
7+
uv venv
8+
uv pip install pre-commit
89
pre-commit install --hook-type pre-commit --hook-type pre-push
9-
pip install '.[dev]'
10-
make compile-protos-python
11-
make compile-protos-go
12-
make compile-go-lib
13-
env:
14-
PYTHONUSERBASE: "/workspace/.pip-modules"
15-
command: |
16-
source venv/bin/activate
17-
18-
git config --global alias.ci 'commit -s'
19-
git config --global alias.sw switch
20-
git config --global alias.st status
21-
git config --global alias.co checkout
22-
git config --global alias.br branch
23-
git config --global alias.df diff
24-
github:
25-
prebuilds:
26-
# enable for the default branch (defaults to true)
27-
master: true
28-
# enable for all branches in this repo (defaults to false)
29-
branches: false
30-
# enable for pull requests coming from this repo (defaults to true)
31-
pullRequests: true
32-
# enable for pull requests coming from forks (defaults to false)
33-
pullRequestsFromForks: false
34-
# add a check to pull requests (defaults to true)
35-
addCheck: true
36-
# add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
37-
addComment: false
38-
# add a "Review in Gitpod" button to the pull request's description (defaults to false)
39-
addBadge: false
10+
source .venv/bin/activate
11+
export PYTHON=3.10 && make install-python-ci-dependencies-uv-venv
12+
# git config --global alias.ci 'commit -s'
13+
# git config --global alias.sw switch
14+
# git config --global alias.st status
15+
# git config --global alias.co checkout
16+
# git config --global alias.br branch
17+
# git config --global alias.df diff
4018
4119
vscode:
4220
extensions:

0 commit comments

Comments
 (0)