Skip to content

Commit 6bf8df0

Browse files
pyalexpyalex
authored
chore: Split single package into sub package in Go feature server (#2513)
* split feast package into subpackages Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * test unpacking request Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * migrate to arrow/v8 to read parquet files written by python Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
1 parent c42c9b0 commit 6bf8df0

29 files changed

+1305
-1326
lines changed

‎go.mod‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/feast-dev/feast
33
go 1.17
44

55
require (
6-
github.com/apache/arrow/go/v7 v7.0.0
6+
github.com/apache/arrow/go/v8 v8.0.0-20220408212425-58fe60f59289
77
github.com/ghodss/yaml v1.0.0
88
github.com/go-python/gopy v0.4.0
99
github.com/go-redis/redis/v8 v8.11.4
@@ -12,32 +12,36 @@ require (
1212
github.com/mattn/go-sqlite3 v1.14.12
1313
github.com/spaolacci/murmur3 v1.1.0
1414
github.com/stretchr/testify v1.7.0
15-
github.com/xitongsys/parquet-go v1.6.2
16-
github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c
1715
google.golang.org/grpc v1.44.0
1816
google.golang.org/protobuf v1.27.1
1917
)
2018

2119
require (
22-
github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
23-
github.com/apache/thrift v0.16.0 // indirect
20+
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
21+
github.com/andybalholm/brotli v1.0.4 // indirect
22+
github.com/apache/thrift v0.15.0 // indirect
2423
github.com/cespare/xxhash/v2 v2.1.2 // indirect
2524
github.com/davecgh/go-spew v1.1.1 // indirect
2625
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
2726
github.com/goccy/go-json v0.7.10 // indirect
2827
github.com/golang/snappy v0.0.4 // indirect
29-
github.com/google/flatbuffers v2.0.0+incompatible // indirect
30-
github.com/google/go-cmp v0.5.7 // indirect
28+
github.com/google/flatbuffers v2.0.5+incompatible // indirect
29+
github.com/klauspost/asmfmt v1.3.1 // indirect
3130
github.com/klauspost/compress v1.15.1 // indirect
32-
github.com/pierrec/lz4/v4 v4.1.9 // indirect
31+
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
32+
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
33+
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
34+
github.com/pierrec/lz4/v4 v4.1.12 // indirect
3335
github.com/pmezard/go-difflib v1.0.0 // indirect
34-
golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4 // indirect
35-
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
36-
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
36+
github.com/zeebo/xxh3 v1.0.1 // indirect
37+
golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 // indirect
38+
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
39+
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
40+
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
3741
golang.org/x/text v0.3.7 // indirect
38-
golang.org/x/tools v0.1.4 // indirect
42+
golang.org/x/tools v0.1.9 // indirect
3943
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
40-
google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect
44+
google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect
4145
gopkg.in/yaml.v2 v2.4.0 // indirect
4246
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
4347
)

‎go.sum‎

Lines changed: 41 additions & 234 deletions
Large diffs are not rendered by default.

‎go/cmd/server/main.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"fmt"
55
"github.com/feast-dev/feast/go/internal/feast"
6+
"github.com/feast-dev/feast/go/internal/feast/registry"
67
"github.com/feast-dev/feast/go/protos/feast/serving"
78
"google.golang.org/grpc"
89
"log"
@@ -26,15 +27,15 @@ func main() {
2627
log.Fatalln(fmt.Sprintf("One of %s of %s environment variables must be set", flagFeastRepoPath, flagFeastRepoConfig))
2728
}
2829

29-
var repoConfig *feast.RepoConfig
30+
var repoConfig *registry.RepoConfig
3031
var err error
3132
if repoConfigJSON != "" {
32-
repoConfig, err = feast.NewRepoConfigFromJSON(repoPath, repoConfigJSON)
33+
repoConfig, err = registry.NewRepoConfigFromJSON(repoPath, repoConfigJSON)
3334
if err != nil {
3435
log.Fatalln(err)
3536
}
3637
} else {
37-
repoConfig, err = feast.NewRepoConfigFromFile(repoPath)
38+
repoConfig, err = registry.NewRepoConfigFromFile(repoPath)
3839
if err != nil {
3940
log.Fatalln(err)
4041
}

‎go/cmd/server/server_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"context"
5+
"github.com/feast-dev/feast/go/internal/feast/registry"
56
"net"
67
"path/filepath"
78
"reflect"
@@ -37,7 +38,7 @@ func getClient(ctx context.Context, basePath string) (serving.ServingServiceClie
3738
listener := bufconn.Listen(buffer)
3839

3940
server := grpc.NewServer()
40-
config, err := feast.NewRepoConfigFromFile(getRepoPath(basePath))
41+
config, err := registry.NewRepoConfigFromFile(getRepoPath(basePath))
4142
if err != nil {
4243
panic(err)
4344
}
@@ -123,9 +124,9 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) {
123124
expectedAvgDailyTripsValues := []*types.Value{}
124125

125126
for _, key := range []int64{1001, 1003, 1005} {
126-
expectedConvRateValues = append(expectedConvRateValues, &types.Value{Val: &types.Value_FloatVal{FloatVal: correctFeatures[key].Conv_rate}})
127-
expectedAccRateValues = append(expectedAccRateValues, &types.Value{Val: &types.Value_FloatVal{FloatVal: correctFeatures[key].Acc_rate}})
128-
expectedAvgDailyTripsValues = append(expectedAvgDailyTripsValues, &types.Value{Val: &types.Value_Int64Val{Int64Val: int64(correctFeatures[key].Avg_daily_trips)}})
127+
expectedConvRateValues = append(expectedConvRateValues, &types.Value{Val: &types.Value_FloatVal{FloatVal: correctFeatures[key].ConvRate}})
128+
expectedAccRateValues = append(expectedAccRateValues, &types.Value{Val: &types.Value_FloatVal{FloatVal: correctFeatures[key].AccRate}})
129+
expectedAvgDailyTripsValues = append(expectedAvgDailyTripsValues, &types.Value{Val: &types.Value_Int64Val{Int64Val: int64(correctFeatures[key].AvgDailyTrips)}})
129130
}
130131
// Columnar so get in column format row by row should have column names of all features
131132
assert.Equal(t, len(response.Results), 4)

‎go/embedded/online_features.go‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,15 @@ package embedded
33
import (
44
"context"
55
"fmt"
6-
"github.com/apache/arrow/go/v7/arrow"
7-
"github.com/apache/arrow/go/v7/arrow/array"
8-
"github.com/apache/arrow/go/v7/arrow/cdata"
9-
"github.com/apache/arrow/go/v7/arrow/memory"
6+
"github.com/apache/arrow/go/v8/arrow"
7+
"github.com/apache/arrow/go/v8/arrow/array"
8+
"github.com/apache/arrow/go/v8/arrow/cdata"
9+
"github.com/apache/arrow/go/v8/arrow/memory"
1010
"github.com/feast-dev/feast/go/internal/feast"
11+
"github.com/feast-dev/feast/go/internal/feast/model"
12+
"github.com/feast-dev/feast/go/internal/feast/onlineserving"
13+
"github.com/feast-dev/feast/go/internal/feast/registry"
14+
"github.com/feast-dev/feast/go/internal/feast/transformation"
1115
prototypes "github.com/feast-dev/feast/go/protos/feast/types"
1216
"github.com/feast-dev/feast/go/types"
1317
"log"
@@ -27,8 +31,8 @@ type DataTable struct {
2731
SchemaPtr uintptr
2832
}
2933

30-
func NewOnlineFeatureService(conf *OnlineFeatureServiceConfig, transformationCallback feast.TransformationCallback) *OnlineFeatureService {
31-
repoConfig, err := feast.NewRepoConfigFromJSON(conf.RepoPath, conf.RepoConfig)
34+
func NewOnlineFeatureService(conf *OnlineFeatureServiceConfig, transformationCallback transformation.TransformationCallback) *OnlineFeatureService {
35+
repoConfig, err := registry.NewRepoConfigFromJSON(conf.RepoPath, conf.RepoConfig)
3236
if err != nil {
3337
log.Fatalln(err)
3438
}
@@ -44,15 +48,15 @@ func NewOnlineFeatureService(conf *OnlineFeatureServiceConfig, transformationCal
4448
func (s *OnlineFeatureService) GetEntityTypesMap(featureRefs []string) (map[string]int32, error) {
4549
viewNames := make(map[string]interface{})
4650
for _, featureRef := range featureRefs {
47-
viewName, _, err := feast.ParseFeatureReference(featureRef)
51+
viewName, _, err := onlineserving.ParseFeatureReference(featureRef)
4852
if err != nil {
4953
return nil, err
5054
}
5155
viewNames[viewName] = nil
5256
}
5357

5458
entities, _ := s.fs.ListEntities(true)
55-
entitiesByName := make(map[string]*feast.Entity)
59+
entitiesByName := make(map[string]*model.Entity)
5660
for _, entity := range entities {
5761
entitiesByName[entity.Name] = entity
5862
}
@@ -83,7 +87,7 @@ func (s *OnlineFeatureService) GetEntityTypesMapByFeatureService(featureServiceN
8387
joinKeyTypes := make(map[string]int32)
8488

8589
entities, _ := s.fs.ListEntities(true)
86-
entitiesByName := make(map[string]*feast.Entity)
90+
entitiesByName := make(map[string]*model.Entity)
8791
for _, entity := range entities {
8892
entitiesByName[entity.Name] = entity
8993
}
@@ -133,7 +137,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures(
133137
return err
134138
}
135139

136-
var featureService *feast.FeatureService
140+
var featureService *model.FeatureService
137141
if featureServiceName != "" {
138142
featureService, err = s.fs.GetFeatureService(featureServiceName)
139143
}

0 commit comments

Comments
 (0)