Skip to content

Commit ddb3f0a

Browse files
TremaMiguelTremaMiguel
committed
Merge branch 'feat/dynamo_db_online_write_read' of github.com:TremaMiguel/feast into feat/dynamo_db_online_write_read
2 parents 97fd71f + 29b5cf6 commit ddb3f0a

122 files changed

Lines changed: 3778 additions & 24127 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/linter.yml‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ jobs:
1515
with:
1616
python-version: "3.7"
1717
architecture: x64
18+
- name: Setup Go
19+
id: setup-go
20+
uses: actions/setup-go@v2
21+
with:
22+
go-version: 1.17.7
1823
- name: Upgrade pip version
1924
run: |
2025
pip install --upgrade "pip>=21.3.1"
@@ -35,16 +40,30 @@ jobs:
3540
- name: Install pip-tools
3641
run: pip install pip-tools
3742
- name: Install dependencies
38-
run: make install-python-ci-dependencies
43+
run: |
44+
make compile-protos-go
45+
make install-python-ci-dependencies
3946
- name: Lint python
4047
run: make lint-python
4148

4249
lint-go:
43-
container: gcr.io/kf-feast/feast-ci:latest
44-
runs-on: [ubuntu-latest]
50+
runs-on: ubuntu-latest
4551
steps:
4652
- uses: actions/checkout@v2
53+
- name: Setup Go
54+
id: setup-go
55+
uses: actions/setup-go@v2
56+
with:
57+
go-version: 1.17.7
58+
- name: Setup Python
59+
id: setup-python
60+
uses: actions/setup-python@v2
61+
with:
62+
python-version: 3.7
63+
- name: Upgrade pip version
64+
run: |
65+
pip install --upgrade "pip>=21.3.1"
4766
- name: Install dependencies
48-
run: make install-go-ci-dependencies
67+
run: make install-go-ci-dependencies
4968
- name: Lint go
5069
run: make lint-go

‎.github/workflows/master_only.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,14 @@ jobs:
5454
docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
5555
outputs:
5656
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
57-
integration-test-python:
57+
integration-test-python-and-go:
5858
needs: build-lambda-docker-image
5959
runs-on: ${{ matrix.os }}
6060
strategy:
6161
fail-fast: false
6262
matrix:
6363
python-version: [ 3.7, 3.8, 3.9 ]
64+
go-version: [ 1.17.0 ]
6465
os: [ ubuntu-latest ]
6566
env:
6667
OS: ${{ matrix.os }}
@@ -73,6 +74,11 @@ jobs:
7374
with:
7475
python-version: ${{ matrix.python-version }}
7576
architecture: x64
77+
- name: Setup Go
78+
id: setup-go
79+
uses: actions/setup-go@v2
80+
with:
81+
go-version: ${{ matrix.go-version }}
7682
- name: Set up Cloud SDK
7783
uses: google-github-actions/setup-gcloud@v0
7884
with:
@@ -113,13 +119,12 @@ jobs:
113119
- name: Start Redis
114120
uses: supercharge/redis-github-action@1.4.0
115121
with:
116-
redis-version: ${{ matrix.redis-version }}
117122
redis-port: 12345
118123
- name: Setup Redis Cluster
119124
run: |
120125
docker pull vishnunair/docker-redis-cluster:latest
121126
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
122-
- name: Test python
127+
- name: Test python and go
123128
env:
124129
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
125130
FEAST_USAGE: "False"

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,9 @@ jobs:
144144
- name: Install pip-tools
145145
run: pip install pip-tools
146146
- name: Install dependencies
147-
run: make install-python-ci-dependencies
147+
run: |
148+
make compile-protos-go
149+
make install-python-ci-dependencies
148150
- name: Setup Redis Cluster
149151
run: |
150152
docker pull vishnunair/docker-redis-cluster:latest

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ jobs:
4040
- name: Release
4141
if: github.event.inputs.dry_run == 'false'
4242
run: |
43-
npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release
43+
npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release

‎.github/workflows/unit_tests.yml‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ jobs:
2020
with:
2121
python-version: ${{ matrix.python-version }}
2222
architecture: x64
23+
- name: Setup Go
24+
id: setup-go
25+
uses: actions/setup-go@v2
26+
with:
27+
go-version: 1.17.7
2328
- name: Upgrade pip version
2429
run: |
2530
pip install --upgrade "pip>=21.3.1"
@@ -40,7 +45,9 @@ jobs:
4045
- name: Install pip-tools
4146
run: pip install pip-tools
4247
- name: Install dependencies
43-
run: make install-python-ci-dependencies
48+
run: |
49+
make compile-protos-go
50+
make install-python-ci-dependencies
4451
- name: Test Python
4552
env:
4653
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
@@ -61,9 +68,21 @@ jobs:
6168

6269
unit-test-go:
6370
runs-on: ubuntu-latest
64-
container: gcr.io/kf-feast/feast-ci:latest
6571
steps:
6672
- uses: actions/checkout@v2
73+
- name: Setup Python
74+
id: setup-python
75+
uses: actions/setup-python@v2
76+
with:
77+
python-version: 3.7
78+
- name: Upgrade pip version
79+
run: |
80+
pip install --upgrade "pip>=21.3.1"
81+
- name: Setup Go
82+
id: setup-go
83+
uses: actions/setup-go@v2
84+
with:
85+
go-version: 1.17.7
6786
- name: Install dependencies
6887
run: make install-go-ci-dependencies
6988
- name: Compile protos

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ dmypy.json
186186
sdk/python/docs/html
187187
sdk/python/feast/protos/
188188
sdk/go/protos/
189+
go/protos/
189190

190191
#benchmarks
191192
.benchmarks
@@ -218,3 +219,5 @@ ui/.vercel
218219
**/yarn-debug.log*
219220
**/yarn-error.log*
220221

222+
# Go subprocess binaries (built during feast pip package building)
223+
sdk/python/feast/binaries/

‎.releaserc.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,3 @@ module.exports = {
7474
}],
7575
]
7676
}
77-

‎Makefile‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616

1717
ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
1818
MVN := mvn -f java/pom.xml ${MAVEN_EXTRA_OPTS}
19-
PROTO_TYPE_SUBDIRS = core serving types storage
20-
PROTO_SERVICE_SUBDIRS = core serving
2119
OS := linux
2220
ifeq ($(shell uname -s), Darwin)
2321
OS = osx
@@ -35,13 +33,11 @@ protos: compile-protos-go compile-protos-python compile-protos-docs
3533

3634
build: protos build-java build-docker build-html
3735

38-
install-ci-dependencies: install-python-ci-dependencies install-java-ci-dependencies install-go-ci-dependencies
39-
4036
# Python SDK
4137

42-
install-python-ci-dependencies:
38+
install-python-ci-dependencies: install-go-ci-dependencies
4339
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
44-
cd sdk/python && python setup.py develop
40+
cd sdk/python && COMPILE_GO=true python setup.py develop
4541

4642
lock-python-ci-dependencies:
4743
cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
@@ -50,8 +46,7 @@ package-protos:
5046
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
5147

5248
compile-protos-python:
53-
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
54-
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -I@ sed -i.bak 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
49+
python setup.py build_python_protos
5550

5651
install-python:
5752
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
@@ -72,12 +67,23 @@ test-python:
7267
test-python-integration:
7368
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
7469

70+
test-python-universal-contrib:
71+
PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
72+
7573
test-python-universal-local:
7674
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
7775

7876
test-python-universal:
7977
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
8078

79+
test-python-go-server:
80+
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
81+
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserver sdk/python/tests
82+
83+
test-python-go-server-lifecycle:
84+
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
85+
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserverlifecycle sdk/python/tests
86+
8187
format-python:
8288
# Sort
8389
cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
@@ -120,20 +126,25 @@ build-java-no-tests:
120126
# Go SDK
121127

122128
install-go-ci-dependencies:
123-
go get -u github.com/golang/protobuf/protoc-gen-go
124-
go get -u golang.org/x/lint/golint
129+
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
130+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
131+
132+
compile-protos-go: install-go-ci-dependencies
133+
pip install grpcio-tools==1.34.0
134+
python sdk/python/setup.py build_go_protos
125135

126-
compile-protos-go:
127-
$(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;)
136+
compile-go-feature-server: compile-protos-go
137+
go mod tidy
138+
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
128139

129-
test-go:
130-
cd ${ROOT_DIR}/sdk/go; go test ./...
140+
test-go: install-go-ci-dependencies
141+
go test ./...
131142

132143
format-go:
133-
cd ${ROOT_DIR}/sdk/go; gofmt -s -w *.go
144+
gofmt -s -w go/
134145

135-
lint-go:
136-
cd ${ROOT_DIR}/sdk/go; go vet
146+
lint-go: compile-protos-go
147+
go vet ./go/internal/feast ./go/cmd/goserver
137148

138149
# Docker
139150

@@ -198,4 +209,4 @@ build-sphinx: compile-protos-python
198209
cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source
199210

200211
build-templates:
201-
python infra/scripts/compile-templates.py
212+
python infra/scripts/compile-templates.py

‎docs/SUMMARY.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,13 @@
6060
* [Snowflake](reference/data-sources/snowflake.md)
6161
* [BigQuery](reference/data-sources/bigquery.md)
6262
* [Redshift](reference/data-sources/redshift.md)
63+
* [Spark](reference/data-sources/spark.md)
6364
* [Offline stores](reference/offline-stores/README.md)
6465
* [File](reference/offline-stores/file.md)
6566
* [Snowflake](reference/offline-stores/snowflake.md)
6667
* [BigQuery](reference/offline-stores/bigquery.md)
6768
* [Redshift](reference/offline-stores/redshift.md)
69+
* [Spark](reference/offline-stores/spark.md)
6870
* [Online stores](reference/online-stores/README.md)
6971
* [SQLite](reference/online-stores/sqlite.md)
7072
* [Redis](reference/online-stores/redis.md)

‎docs/getting-started/architecture-and-components/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
{% page-ref page="feature-repository.md" %}
66

7-
{% page-ref page="untitled.md" %}
7+
{% page-ref page="registry.md" %}
88

99
{% page-ref page="offline-store.md" %}
1010

0 commit comments

Comments
 (0)