Skip to content

Commit cb7d8c4

Browse files
shuchudharmisha
shuchu
authored and
dharmisha
committed
feat: Update the go feature server from Expedia code repo. (feast-dev#4665)
* feat: Update the go feature server from Expedia code repo. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Add go package definition to RegistryServer and Grpcserver. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Fix the make build-go Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Fix makefile to make test-go work. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Removed and commented out DataDog related observability instrumentation code. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * doc: Update the README to mention the contirbution from Expedia Group. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Remove the HTTP based Registry. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Use a general string to represent the transformation service endpoint. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Set the transformation service endpoint defintion to feature_store.yaml file instead of OS env. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> * fix: Fix few format issues. Signed-off-by: Shuchu Han <shuchu.han@gmail.com> --------- Signed-off-by: Shuchu Han <shuchu.han@gmail.com>
1 parent 298059f commit cb7d8c4

4 files changed

Lines changed: 184 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,7 @@ build-ui:
586586

587587

588588
# Go SDK & embedded
589+
<<<<<<< HEAD
589590
PB_REL = https://github.com/protocolbuffers/protobuf/releases
590591
PB_VERSION = 3.11.2
591592
PB_ARCH := $(shell uname -m)
@@ -613,10 +614,20 @@ compile-protos-go: install-go-proto-dependencies
613614
--go-grpc_out=$(ROOT_DIR)/go/protos \
614615
--go-grpc_opt=module=github.com/feast-dev/feast/go/protos $(ROOT_DIR)/protos/feast/$(folder)/*.proto; ) true
615616

617+
=======
618+
install-protoc-dependencies:
619+
pip install "protobuf>=4.24.0,<5.0.0" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
620+
621+
install-go-proto-dependencies:
622+
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0
623+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
624+
625+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
616626
#install-go-ci-dependencies:
617627
# go install golang.org/x/tools/cmd/goimports
618628
# python -m pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
619629

630+
<<<<<<< HEAD
620631
.PHONY: build-go
621632
build-go: compile-protos-go
622633
go build -o feast ./go/main.go
@@ -638,8 +649,37 @@ lint-go: compile-protos-go
638649
go vet ./go/internal/feast
639650

640651
.PHONY: build-go-docker-dev
652+
=======
653+
build-go:
654+
compile-protos-go
655+
go build -o feast ./go/main.go
656+
657+
install-feast-ci-locally:
658+
pip install -e ".[ci]"
659+
660+
test-go:
661+
compile-protos-go
662+
compile-protos-python
663+
install-feast-ci-locally
664+
CGO_ENABLED=1 go test -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html
665+
666+
format-go:
667+
gofmt -s -w go/
668+
669+
lint-go:
670+
compile-protos-go
671+
go vet ./go/internal/feast
672+
673+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
641674
build-go-docker-dev:
642675
docker buildx build --build-arg VERSION=dev \
643676
-t feastdev/feature-server-go:dev \
644677
-f go/infra/docker/feature-server/Dockerfile --load .
645678

679+
<<<<<<< HEAD
680+
=======
681+
compile-protos-go:
682+
install-go-proto-dependencies
683+
install-protoc-dependencies
684+
python setup.py build_go_protos
685+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))

‎go/internal/feast/featurestore.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,14 @@ package feast
33
import (
44
"context"
55
"errors"
6+
<<<<<<< HEAD
67

78
"github.com/apache/arrow/go/v17/arrow/memory"
89

10+
=======
11+
"fmt"
12+
"github.com/apache/arrow/go/v17/arrow/memory"
13+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
914
//"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
1015

1116
"github.com/feast-dev/feast/go/internal/feast/model"
@@ -61,13 +66,26 @@ func NewFeatureStore(config *registry.RepoConfig, callback transformation.Transf
6166
return nil, err
6267
}
6368

69+
<<<<<<< HEAD
6470
var transformationService *transformation.GrpcTransformationService
6571
if transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"]; ok {
6672
// Use a scalable transformation service like Python Transformation Service.
6773
// Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file
6874
// under the "feature_server" section.
6975
transformationService, _ = transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string))
7076
}
77+
=======
78+
// Use a scalable transformation service like Python Transformation Service.
79+
// Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file
80+
// under the "feature_server" section.
81+
transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"]
82+
if !ok {
83+
fmt.Println("Errors while reading transformation_service_endpoint info")
84+
panic("No transformation service endpoint provided in the feature_store.yaml file.")
85+
}
86+
87+
transformationService, _ := transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string))
88+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
7189

7290
return &FeatureStore{
7391
config: config,

‎go/internal/feast/featurestore_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func TestNewFeatureStore(t *testing.T) {
9191
errMessage: "invalid_store online store type is currently not supported",
9292
},
9393
}
94+
<<<<<<< HEAD
9495
for _, test := range tests {
9596
t.Run(test.name, func(t *testing.T) {
9697
got, err := NewFeatureStore(test.config, nil)
@@ -107,6 +108,64 @@ func TestNewFeatureStore(t *testing.T) {
107108
})
108109
}
109110

111+
=======
112+
fs, err := NewFeatureStore(&config, nil)
113+
assert.Nil(t, err)
114+
assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore)
115+
116+
t.Run("valid config", func(t *testing.T) {
117+
config := &registry.RepoConfig{
118+
Project: "feature_repo",
119+
Registry: getRegistryPath(),
120+
Provider: "local",
121+
OnlineStore: map[string]interface{}{
122+
"type": "redis",
123+
},
124+
FeatureServer: map[string]interface{}{
125+
"transformation_service_endpoint": "localhost:50051",
126+
},
127+
}
128+
fs, err := NewFeatureStore(config, nil)
129+
assert.Nil(t, err)
130+
assert.NotNil(t, fs)
131+
assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore)
132+
assert.NotNil(t, fs.transformationService)
133+
})
134+
135+
t.Run("missing transformation service endpoint", func(t *testing.T) {
136+
config := &registry.RepoConfig{
137+
Project: "feature_repo",
138+
Registry: getRegistryPath(),
139+
Provider: "local",
140+
OnlineStore: map[string]interface{}{
141+
"type": "redis",
142+
},
143+
}
144+
defer func() {
145+
if r := recover(); r == nil {
146+
t.Errorf("The code did not panic")
147+
}
148+
}()
149+
NewFeatureStore(config, nil)
150+
})
151+
152+
t.Run("invalid online store config", func(t *testing.T) {
153+
config := &registry.RepoConfig{
154+
Project: "feature_repo",
155+
Registry: getRegistryPath(),
156+
Provider: "local",
157+
OnlineStore: map[string]interface{}{
158+
"type": "invalid_store",
159+
},
160+
FeatureServer: map[string]interface{}{
161+
"transformation_service_endpoint": "localhost:50051",
162+
},
163+
}
164+
fs, err := NewFeatureStore(config, nil)
165+
assert.NotNil(t, err)
166+
assert.Nil(t, fs)
167+
})
168+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
110169
}
111170

112171
type MockRedis struct {

‎setup.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,11 @@
156156

157157
GO_REQUIRED = ["cffi~=1.15.0"]
158158

159+
<<<<<<< HEAD
159160
MILVUS_REQUIRED = ["pymilvus"]
160161

162+
=======
163+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
161164
CI_REQUIRED = (
162165
[
163166
"build",
@@ -261,6 +264,64 @@
261264

262265
PYTHON_CODE_PREFIX = "sdk/python"
263266

267+
<<<<<<< HEAD
268+
=======
269+
def _generate_path_with_gopath():
270+
go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8")
271+
go_path = go_path.strip()
272+
path_val = os.getenv("PATH")
273+
path_val = f"{path_val}:{go_path}/bin"
274+
275+
return path_val
276+
277+
class BuildGoProtosCommand(Command):
278+
description = "Builds the proto files into Go files."
279+
user_options = []
280+
281+
def initialize_options(self):
282+
self.go_protoc = [
283+
sys.executable,
284+
"-m",
285+
"grpc_tools.protoc",
286+
] # find_executable("protoc")
287+
self.proto_folder = os.path.join(repo_root, "protos")
288+
self.go_folder = os.path.join(repo_root, "go/protos")
289+
self.sub_folders = ["core", "registry", "serving", "types", "storage"]
290+
self.path_val = _generate_path_with_gopath()
291+
292+
def finalize_options(self):
293+
pass
294+
295+
def _generate_go_protos(self, path: str):
296+
proto_files = glob.glob(os.path.join(self.proto_folder, path))
297+
298+
try:
299+
subprocess.check_call(
300+
self.go_protoc
301+
+ [
302+
"-I",
303+
self.proto_folder,
304+
"--go_out",
305+
self.go_folder,
306+
"--go_opt=module=github.com/feast-dev/feast/go/protos",
307+
"--go-grpc_out",
308+
self.go_folder,
309+
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos",
310+
]
311+
+ proto_files,
312+
env={"PATH": self.path_val},
313+
)
314+
except CalledProcessError as e:
315+
print(f"Stderr: {e.stderr}")
316+
print(f"Stdout: {e.stdout}")
317+
318+
def run(self):
319+
go_dir = Path(repo_root) / "go" / "protos"
320+
go_dir.mkdir(exist_ok=True)
321+
for sub_folder in self.sub_folders:
322+
self._generate_go_protos(f"feast/{sub_folder}/*.proto")
323+
324+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
264325

265326
setup(
266327
name=NAME,
@@ -307,7 +368,10 @@
307368
"faiss": FAISS_REQUIRED,
308369
"qdrant": QDRANT_REQUIRED,
309370
"go": GO_REQUIRED,
371+
<<<<<<< HEAD
310372
"milvus": MILVUS_REQUIRED,
373+
=======
374+
>>>>>>> 96da3d849 (feat: Update the go feature server from Expedia code repo. (#4665))
311375
},
312376
include_package_data=True,
313377
license="Apache",
@@ -325,4 +389,7 @@
325389
"pybindgen==0.22.0", # TODO do we need this?
326390
"setuptools_scm>=6.2", # TODO do we need this?
327391
],
392+
cmdclass={
393+
"build_go_protos": BuildGoProtosCommand
394+
},
328395
)

0 commit comments

Comments
 (0)