Skip to content

Commit 0fbc062

Browse files
pyalexpyalex
authored
chore: Use Go-based implementation as embedded extension to Python SDK (#2429)
* work in progress Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * more supported type for proto <-> arrow converstion Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * better names Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fix arrow to proto conversion Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fix import Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * address some PR comments Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fix Makefile Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * type conversion test Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * rename package utils -> types Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * clean up on renaming Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
1 parent ad008bf commit 0fbc062

25 files changed

+1745
-690
lines changed

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,7 @@ jobs:
151151
- name: Install pip-tools
152152
run: pip install pip-tools
153153
- name: Install dependencies
154-
run: |
155-
make compile-protos-go
156-
make install-python-ci-dependencies
154+
run: make install-python-ci-dependencies
157155
- name: Setup Redis Cluster
158156
run: |
159157
docker pull vishnunair/docker-redis-cluster:latest

‎.github/workflows/unit_tests.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,7 @@ jobs:
4545
- name: Install pip-tools
4646
run: pip install pip-tools
4747
- name: Install dependencies
48-
run: |
49-
make compile-protos-go
50-
make install-python-ci-dependencies
48+
run: make install-python-ci-dependencies
5149
- name: Test Python
5250
env:
5351
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}

‎Makefile‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ build: protos build-java build-docker build-html
3535

3636
# Python SDK
3737

38-
install-python-ci-dependencies: install-go-proto-dependencies
38+
install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
3939
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
4040
cd sdk/python && COMPILE_GO=true python setup.py develop
4141

@@ -76,14 +76,9 @@ test-python-universal-local:
7676
test-python-universal:
7777
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
7878

79-
test-python-go-server:
80-
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
79+
test-python-go-server: compile-go-lib
8180
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserver sdk/python/tests
8281

83-
test-python-go-server-lifecycle:
84-
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
85-
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserverlifecycle sdk/python/tests
86-
8782
format-python:
8883
# Sort
8984
cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
@@ -123,21 +118,28 @@ build-java:
123118
build-java-no-tests:
124119
${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -DskipITs=true -Drevision=${REVISION} clean package
125120

126-
# Go SDK
121+
# Go SDK & embedded
127122

128123
install-go-proto-dependencies:
129124
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
130125
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
131126

127+
install-go-ci-dependencies:
128+
# ToDo: currently gopy installation doesn't work w/o explicit go get in the next two lines
129+
# ToDo: there should be a better way to install gopy
130+
go get golang.org/x/tools/cmd/goimports
131+
go get github.com/go-python/gopy
132+
go install github.com/go-python/gopy
133+
132134
install-protoc-dependencies:
133135
pip install grpcio-tools==1.34.0
134136

135137
compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
136138
cd sdk/python && python setup.py build_go_protos
137139

138-
compile-go-feature-server: compile-protos-go
139-
go mod tidy
140-
go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
140+
compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
141+
python -m pip install pybindgen==0.22.0
142+
cd sdk/python && python setup.py build_go_lib
141143

142144
test-go: compile-protos-go
143145
go test ./...

‎go.mod‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ module github.com/feast-dev/feast
33
go 1.17
44

55
require (
6+
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40
67
github.com/ghodss/yaml v1.0.0
8+
github.com/go-python/gopy v0.4.0
79
github.com/go-redis/redis/v8 v8.11.4
810
github.com/golang/protobuf v1.5.2
911
github.com/google/uuid v1.1.2
@@ -17,12 +19,21 @@ require (
1719
github.com/cespare/xxhash/v2 v2.1.2 // indirect
1820
github.com/davecgh/go-spew v1.1.1 // indirect
1921
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
22+
github.com/google/flatbuffers v2.0.0+incompatible // indirect
2023
github.com/google/go-cmp v0.5.7 // indirect
24+
github.com/klauspost/compress v1.13.6 // indirect
25+
github.com/kr/pretty v0.1.0 // indirect
26+
github.com/pierrec/lz4/v4 v4.1.9 // indirect
2127
github.com/pmezard/go-difflib v1.0.0 // indirect
22-
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
23-
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
24-
golang.org/x/text v0.3.6 // indirect
28+
golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect
29+
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
30+
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
31+
golang.org/x/text v0.3.7 // indirect
32+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
2533
google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect
34+
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
2635
gopkg.in/yaml.v2 v2.4.0 // indirect
2736
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
2837
)
38+
39+
replace github.com/go-python/gopy v0.4.0 => github.com/pyalex/gopy v0.4.1-0.20220315220227-146bcd99e9f5

0 commit comments

Comments
 (0)