Skip to content

Update GetOnlineFeatures method in sdks #1052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 7 commits into from
Oct 15, 2020
Merged

Update GetOnlineFeatures method in sdks #1052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
be192bd
Update java sdk
terryyylim Oct 14, 2020
72b8f5a
Update python sdk
terryyylim Oct 14, 2020
04b04d4
Update go protos
terryyylim Oct 14, 2020
7ab4ca7
Update go sdk
terryyylim Oct 14, 2020
53b0ffc
Allow project to be unspecified for default proj
terryyylim Oct 15, 2020
bd88ec0
Address PR comments
terryyylim Oct 15, 2020
2c266af
Some cleanups
terryyylim Oct 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/net v0.0.0-20200822124328-c89045814202
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect
golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 // indirect
golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 // indirect
google.golang.org/grpc v1.29.1
google.golang.org/protobuf v1.25.0 // indirect
gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,8 @@ golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 h1:BANdcOVw3KTuUiyfDp7wrzC
golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201013053347-2db1cd791039 h1:kLBxO4OPBgPwjg8Vvu+/0DCHIfDwYIGNFcD66NU9kpo=
golang.org/x/tools v0.0.0-20201013053347-2db1cd791039/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 h1:2ntEwh02rqo2jSsrYmp4yKHHjh0CbXP3ZtSUetSB+q8=
golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
Expand Down
2 changes: 1 addition & 1 deletion sdk/go/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures
if err != nil {
return nil, err
}
resp, err := fc.cli.GetOnlineFeatures(ctx, featuresRequest)
resp, err := fc.cli.GetOnlineFeaturesV2(ctx, featuresRequest)

// collect unqiue entity refs from entity rows
entityRefs := make(map[string]struct{})
Expand Down
15 changes: 6 additions & 9 deletions sdk/go/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ func TestGetOnlineFeatures(t *testing.T) {
req: OnlineFeaturesRequest{
Features: []string{
"driver:rating",
"rating",
"null_value",
"driver:null_value",
},
Entities: []Row{
{"driver_id": Int64Val(1)},
Expand All @@ -39,14 +38,12 @@ func TestGetOnlineFeatures(t *testing.T) {
FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{
{
Fields: map[string]*types.Value{
"driver:rating": Int64Val(1),
"rating": Int64Val(1),
"null_value": {},
"driver:rating": Int64Val(1),
"driver:null_value": {},
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"driver:rating": serving.GetOnlineFeaturesResponse_PRESENT,
"rating": serving.GetOnlineFeaturesResponse_PRESENT,
"null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE,
"driver:rating": serving.GetOnlineFeaturesResponse_PRESENT,
"driver:null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE,
},
},
},
Expand All @@ -65,7 +62,7 @@ func TestGetOnlineFeatures(t *testing.T) {
_, traceCtx := opentracing.StartSpanFromContext(ctx, "get_online_features")
rawRequest, _ := tc.req.buildRequest()
resp := tc.want.RawResponse
cli.EXPECT().GetOnlineFeatures(traceCtx, rawRequest).Return(resp, nil).Times(1)
cli.EXPECT().GetOnlineFeaturesV2(traceCtx, rawRequest).Return(resp, nil).Times(1)

client := &GrpcClient{
cli: cli,
Expand Down
155 changes: 84 additions & 71 deletions sdk/go/protos/feast/core/DataSource.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

54 changes: 18 additions & 36 deletions sdk/go/request.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ var (
// ErrInvalidFeatureRef indicates that the user has provided a feature reference
// with the wrong structure or contents
ErrInvalidFeatureRef = "Invalid Feature Reference %s provided, " +
"feature reference must be in the format [featureset:]name"
"feature reference must be in the format featureTableName:featureName"
)

// OnlineFeaturesRequest wrapper on feast.serving.GetOnlineFeaturesRequest.
// OnlineFeaturesRequest wrapper on feast.serving.GetOnlineFeaturesRequestV2.
type OnlineFeaturesRequest struct {
// Features is the list of features to obtain from Feast. Each feature can be given as
// the format feature_set:feature, where "feature_set" & "feature" are feature set name
// the format feature_table:feature, where "feature_table" & "feature" are feature table name
// and feature name respectively. The only required components is feature name.
Features []string

Expand All @@ -26,41 +26,37 @@ type OnlineFeaturesRequest struct {
// Project optionally specifies the project override. If specified, uses given project for retrieval.
// Overrides the projects specified in Feature References if also specified.
Project string

// whether to omit the entities fields in the response.
OmitEntities bool
}

// Builds the feast-specified request payload from the wrapper.
func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest, error) {
func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequestV2, error) {
featureRefs, err := buildFeatureRefs(r.Features)
if err != nil {
return nil, err
}

// build request entity rows from native entities
entityRows := make([]*serving.GetOnlineFeaturesRequest_EntityRow, len(r.Entities))
entityRows := make([]*serving.GetOnlineFeaturesRequestV2_EntityRow, len(r.Entities))
for i, entity := range r.Entities {
entityRows[i] = &serving.GetOnlineFeaturesRequest_EntityRow{
entityRows[i] = &serving.GetOnlineFeaturesRequestV2_EntityRow{
Fields: entity,
}
}

return &serving.GetOnlineFeaturesRequest{
Features: featureRefs,
EntityRows: entityRows,
OmitEntitiesInResponse: r.OmitEntities,
Project: r.Project,
return &serving.GetOnlineFeaturesRequestV2{
Features: featureRefs,
EntityRows: entityRows,
Project: r.Project,
}, nil
}

// Creates a slice of FeatureReferences from string representation in
// the format featureset:feature.
// the format featuretable:feature.
// featureRefStrs - string feature references to parse.
// Returns parsed FeatureReferences.
// Returns an error when the format of the string feature reference is invalid
func buildFeatureRefs(featureRefStrs []string) ([]*serving.FeatureReference, error) {
var featureRefs []*serving.FeatureReference
func buildFeatureRefs(featureRefStrs []string) ([]*serving.FeatureReferenceV2, error) {
var featureRefs []*serving.FeatureReferenceV2

for _, featureRefStr := range featureRefStrs {
featureRef, err := parseFeatureRef(featureRefStr)
Expand All @@ -76,35 +72,21 @@ func buildFeatureRefs(featureRefStrs []string) ([]*serving.FeatureReference, err
// featureRefStr - the string feature reference to parse.
// Returns parsed FeatureReference.
// Returns an error when the format of the string feature reference is invalid
func parseFeatureRef(featureRefStr string) (*serving.FeatureReference, error) {
func parseFeatureRef(featureRefStr string) (*serving.FeatureReferenceV2, error) {
if len(featureRefStr) == 0 {
return nil, fmt.Errorf(ErrInvalidFeatureRef, featureRefStr)
}

var featureRef serving.FeatureReference
if strings.Contains(featureRefStr, "/") {
var featureRef serving.FeatureReferenceV2
if strings.Contains(featureRefStr, "/") || !strings.Contains(featureRefStr, ":") {
return nil, fmt.Errorf(ErrInvalidFeatureRef, featureRefStr)
}
// parse featureset if specified
// parse featuretable if specified
if strings.Contains(featureRefStr, ":") {
refSplit := strings.Split(featureRefStr, ":")
featureRef.FeatureSet, featureRefStr = refSplit[0], refSplit[1]
featureRef.FeatureTable, featureRefStr = refSplit[0], refSplit[1]
}
featureRef.Name = featureRefStr

return &featureRef, nil
}

// Converts a FeatureReference proto into a string
// featureRef - The FeatureReference to render as string
// Returns string representation of the given FeatureReference
func toFeatureRefStr(featureRef *serving.FeatureReference) string {
refStr := ""
// In protov3, unset string and default to ""
if len(featureRef.FeatureSet) > 0 {
refStr += featureRef.FeatureSet + ":"
}
refStr += featureRef.Name

return refStr
}
29 changes: 17 additions & 12 deletions sdk/go/request_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ func TestGetOnlineFeaturesRequest(t *testing.T) {
tt := []struct {
name string
req OnlineFeaturesRequest
want *serving.GetOnlineFeaturesRequest
want *serving.GetOnlineFeaturesRequestV2
wantErr bool
err error
}{
Expand All @@ -22,7 +22,6 @@ func TestGetOnlineFeaturesRequest(t *testing.T) {
req: OnlineFeaturesRequest{
Features: []string{
"driver:driver_id",
"driver_id",
},
Entities: []Row{
{"entity1": Int64Val(1), "entity2": StrVal("bob")},
Expand All @@ -31,17 +30,14 @@ func TestGetOnlineFeaturesRequest(t *testing.T) {
},
Project: "driver_project",
},
want: &serving.GetOnlineFeaturesRequest{
Features: []*serving.FeatureReference{
want: &serving.GetOnlineFeaturesRequestV2{
Features: []*serving.FeatureReferenceV2{
{
FeatureSet: "driver",
Name: "driver_id",
},
{
Name: "driver_id",
FeatureTable: "driver",
Name: "driver_id",
},
},
EntityRows: []*serving.GetOnlineFeaturesRequest_EntityRow{
EntityRows: []*serving.GetOnlineFeaturesRequestV2_EntityRow{
{
Fields: map[string]*types.Value{
"entity1": Int64Val(1),
Expand All @@ -61,8 +57,7 @@ func TestGetOnlineFeaturesRequest(t *testing.T) {
},
},
},
OmitEntitiesInResponse: false,
Project: "driver_project",
Project: "driver_project",
},
wantErr: false,
err: nil,
Expand All @@ -77,6 +72,16 @@ func TestGetOnlineFeaturesRequest(t *testing.T) {
wantErr: true,
err: fmt.Errorf(ErrInvalidFeatureRef, "/fs1:feature1"),
},
{
name: "invalid_feature_name",
req: OnlineFeaturesRequest{
Features: []string{"feature1"},
Entities: []Row{},
Project: "my_project",
},
wantErr: true,
err: fmt.Errorf(ErrInvalidFeatureRef, "feature1"),
},
}
for _, tc := range tt {
t.Run(tc.name, func(t *testing.T) {
Expand Down
36 changes: 18 additions & 18 deletions sdk/go/response_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,22 @@ var response = OnlineFeaturesResponse{
FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{
{
Fields: map[string]*types.Value{
"project1/feature1": Int64Val(1),
"project1/feature2": {},
"featuretable1:feature1": Int64Val(1),
"featuretable1:feature2": {},
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE,
"featuretable1:feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE,
},
},
{
Fields: map[string]*types.Value{
"project1/feature1": Int64Val(2),
"project1/feature2": Int64Val(2),
"featuretable1:feature1": Int64Val(2),
"featuretable1:feature2": Int64Val(2),
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature2": serving.GetOnlineFeaturesResponse_PRESENT,
},
},
},
Expand All @@ -38,8 +38,8 @@ var response = OnlineFeaturesResponse{
func TestOnlineFeaturesResponseToRow(t *testing.T) {
actual := response.Rows()
expected := []Row{
{"project1/feature1": Int64Val(1), "project1/feature2": &types.Value{}},
{"project1/feature1": Int64Val(2), "project1/feature2": Int64Val(2)},
{"featuretable1:feature1": Int64Val(1), "featuretable1:feature2": &types.Value{}},
{"featuretable1:feature1": Int64Val(2), "featuretable1:feature2": Int64Val(2)},
}
if len(expected) != len(actual) {
t.Errorf("expected: %v, got: %v", expected, actual)
Expand All @@ -55,12 +55,12 @@ func TestOnlineFeaturesResponseoToStatuses(t *testing.T) {
actual := response.Statuses()
expected := []map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
{
"project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE,
"featuretable1:feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE,
},
{
"project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature1": serving.GetOnlineFeaturesResponse_PRESENT,
"featuretable1:feature2": serving.GetOnlineFeaturesResponse_PRESENT,
},
}
if len(expected) != len(actual) {
Expand Down Expand Up @@ -88,7 +88,7 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) {
{
name: "valid",
args: args{
order: []string{"project1/feature2", "project1/feature1"},
order: []string{"featuretable1:feature2", "featuretable1:feature1"},
fillNa: []int64{-1, -1},
},
want: [][]int64{{-1, 1}, {2, 2}},
Expand All @@ -97,7 +97,7 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) {
{
name: "length mismatch",
args: args{
order: []string{"fs:feature2", "fs:feature1"},
order: []string{"ft:feature2", "ft:feature1"},
fillNa: []int64{-1},
},
want: nil,
Expand All @@ -107,12 +107,12 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) {
{
name: "length mismatch",
args: args{
order: []string{"project1/feature2", "project1/feature3"},
order: []string{"featuretable1:feature2", "featuretable1:feature3"},
fillNa: []int64{-1, -1},
},
want: nil,
wantErr: true,
err: fmt.Errorf(ErrFeatureNotFound, "project1/feature3"),
err: fmt.Errorf(ErrFeatureNotFound, "featuretable1:feature3"),
},
}
for _, tc := range tt {
Expand Down
Loading