Skip to content

chore: Use Go-based implementation as embedded extension to Python SDK #2429

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 10 commits into from
Mar 24, 2022
Merged

chore: Use Go-based implementation as embedded extension to Python SDK #2429

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
8961766
work in progress
pyalex Mar 16, 2022
8b03be6
more supported type for proto <-> arrow converstion
pyalex Mar 23, 2022
47cae14
better names
pyalex Mar 23, 2022
70bdf1a
fix arrow to proto conversion
pyalex Mar 23, 2022
d14eeb2
fix import
pyalex Mar 23, 2022
6ed2a2a
address some PR comments
pyalex Mar 23, 2022
dfd3a98
fix Makefile
pyalex Mar 24, 2022
ba40a15
type conversion test
pyalex Mar 24, 2022
167c07a
rename package utils -> types
pyalex Mar 24, 2022
5f97c4b
clean up on renaming
pyalex Mar 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
type conversion test 
Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
  • Loading branch information
@pyalex
pyalex committed Mar 24, 2022
commit ba40a153db7db6318872df843f274686dc740499
28 changes: 4 additions & 24 deletions go/internal/feast/featurestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"sort"
"strings"

"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/feast-dev/feast/go/protos/feast/serving"
Expand Down Expand Up @@ -622,30 +621,11 @@ func (fs *FeatureStore) transposeResponseIntoColumns(featureData2D [][]FeatureDa
currentVector.Timestamps[rowIndex] = eventTimeStamp
}
}
var fieldType arrow.DataType
var err error

for _, val := range protoValues {
if val != nil {
fieldType, err = utils.ProtoTypeToArrowType(val)
if err != nil {
return nil, err
}
break
}
}

if fieldType != nil {
builder := array.NewBuilder(arrowAllocator, fieldType)
err = utils.ProtoValuesToArrowArray(builder, protoValues)
if err != nil {
return nil, err
}

currentVector.Values = builder.NewArray()
} else {
currentVector.Values = array.NewNull(numRows)
arrowValues, err := utils.ProtoValuesToArrowArray(protoValues, arrowAllocator, numRows)
Copy link
Copy Markdown
Member

@achals achals Mar 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit, can we have a different module name instead of utils? Seems like it'll become a kitchen sink..

Copy link
Copy Markdown
Collaborator Author

@pyalex pyalex Mar 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

renamed into types since it's type utils

if err != nil {
return nil, err
}
currentVector.Values = arrowValues
}

return vectors, nil
Expand Down
52 changes: 50 additions & 2 deletions go/utils/typeconversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/feast-dev/feast/go/protos/feast/types"
)

Expand Down Expand Up @@ -47,7 +48,7 @@ func ProtoTypeToArrowType(sample *types.Value) (arrow.DataType, error) {
}
}

func ProtoValuesToArrowArray(builder array.Builder, values []*types.Value) error {
func copyProtoValuesToArrowArray(builder array.Builder, values []*types.Value) error {
switch fieldBuilder := builder.(type) {
case *array.BooleanBuilder:
for _, v := range values {
Expand Down Expand Up @@ -185,6 +186,16 @@ func ArrowValuesToProtoValues(arr array.Interface) ([]*types.Value, error) {
}
values = append(values,
&types.Value{Val: &types.Value_BoolListVal{BoolListVal: &types.BoolList{Val: vals}}})
case arrow.FixedWidthTypes.Time64ns:
vals := make([]int64, int(offsets[idx])-pos)
for j := pos; j < int(offsets[idx]); j++ {
vals[j-pos] = int64(listValues.(*array.Time64).Value(j))
}

values = append(values,
&types.Value{Val: &types.Value_UnixTimestampListVal{
UnixTimestampListVal: &types.Int64List{Val: vals}}})

}

// set the end of current element as start of the next
Expand All @@ -207,6 +218,10 @@ func ArrowValuesToProtoValues(arr array.Interface) ([]*types.Value, error) {
for _, v := range arr.(*array.Float32).Float32Values() {
values = append(values, &types.Value{Val: &types.Value_FloatVal{FloatVal: v}})
}
case arrow.PrimitiveTypes.Float64:
for _, v := range arr.(*array.Float64).Float64Values() {
values = append(values, &types.Value{Val: &types.Value_DoubleVal{DoubleVal: v}})
}
case arrow.FixedWidthTypes.Boolean:
for idx := 0; idx < arr.Len(); idx++ {
values = append(values,
Expand All @@ -220,11 +235,44 @@ func ArrowValuesToProtoValues(arr array.Interface) ([]*types.Value, error) {
case arrow.BinaryTypes.String:
for idx := 0; idx < arr.Len(); idx++ {
values = append(values,
&types.Value{Val: &types.Value_StringVal{StringVal: arr.(*array.Binary).ValueString(idx)}})
&types.Value{Val: &types.Value_StringVal{StringVal: arr.(*array.String).Value(idx)}})
}
case arrow.FixedWidthTypes.Time64ns:
for idx := 0; idx < arr.Len(); idx++ {
values = append(values,
&types.Value{Val: &types.Value_UnixTimestampVal{
UnixTimestampVal: int64(arr.(*array.Time64).Value(idx))}})
}
default:
return nil, fmt.Errorf("unsupported arrow to proto conversion for type %s", arr.DataType())
}

return values, nil
}

func ProtoValuesToArrowArray(protoValues []*types.Value, arrowAllocator memory.Allocator, numRows int) (array.Interface, error) {
var fieldType arrow.DataType
var err error

for _, val := range protoValues {
if val != nil {
fieldType, err = ProtoTypeToArrowType(val)
if err != nil {
return nil, err
}
break
}
}

if fieldType != nil {
builder := array.NewBuilder(arrowAllocator, fieldType)
err = copyProtoValuesToArrowArray(builder, protoValues)
if err != nil {
return nil, err
}

return builder.NewArray(), nil
} else {
return array.NewNull(numRows), nil
}
}
80 changes: 80 additions & 0 deletions go/utils/typeconversion_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package utils

import (
"github.com/apache/arrow/go/arrow/memory"
"github.com/feast-dev/feast/go/protos/feast/types"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"testing"
"time"
)

var (
PROTO_VALUES = [][]*types.Value{
{{Val: &types.Value_Int32Val{10}}, {Val: &types.Value_Int32Val{20}}},
{{Val: &types.Value_Int64Val{10}}, {Val: &types.Value_Int64Val{20}}},
{{Val: &types.Value_FloatVal{1.0}}, {Val: &types.Value_FloatVal{2.0}}},
{{Val: &types.Value_DoubleVal{1.0}}, {Val: &types.Value_DoubleVal{2.0}}},
{{Val: &types.Value_StringVal{"aaa"}}, {Val: &types.Value_StringVal{"bbb"}}},
{{Val: &types.Value_BytesVal{[]byte{1, 2, 3}}}, {Val: &types.Value_BytesVal{[]byte{4, 5, 6}}}},
{{Val: &types.Value_BoolVal{true}}, {Val: &types.Value_BoolVal{false}}},
{{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}},
{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}},

{
{Val: &types.Value_Int32ListVal{&types.Int32List{Val: []int32{0, 1, 2}}}},
{Val: &types.Value_Int32ListVal{&types.Int32List{Val: []int32{3, 4, 5}}}},
},
{
{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{0, 1, 2}}}},
{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{3, 4, 5}}}},
},
{
{Val: &types.Value_FloatListVal{&types.FloatList{Val: []float32{0.5, 1.5, 2}}}},
{Val: &types.Value_FloatListVal{&types.FloatList{Val: []float32{3.5, 4, 5}}}},
},
{
{Val: &types.Value_DoubleListVal{&types.DoubleList{Val: []float64{0.5, 1, 2}}}},
{Val: &types.Value_DoubleListVal{&types.DoubleList{Val: []float64{3.5, 4, 5}}}},
},
{
{Val: &types.Value_BytesListVal{&types.BytesList{Val: [][]byte{{0, 1}, {2}}}}},
{Val: &types.Value_BytesListVal{&types.BytesList{Val: [][]byte{{3, 4}, {5}}}}},
},
{
{Val: &types.Value_StringListVal{&types.StringList{Val: []string{"aa", "bb"}}}},
{Val: &types.Value_StringListVal{&types.StringList{Val: []string{"cc", "dd"}}}},
},
{
{Val: &types.Value_BoolListVal{&types.BoolList{Val: []bool{false, false}}}},
{Val: &types.Value_BoolListVal{&types.BoolList{Val: []bool{true, true}}}},
},
{
{Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix()}}}},
{Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix()}}}},
},
}
)

func TestConversionBetweenProtoAndArrow(t *testing.T) {
pool := memory.NewGoAllocator()
for _, vector := range PROTO_VALUES {
arrowArray, err := ProtoValuesToArrowArray(vector, pool, len(vector))
assert.Nil(t, err)

protoValues, err := ArrowValuesToProtoValues(arrowArray)
assert.Nil(t, err)

protoValuesEquals(t, vector, protoValues)
}

}

func protoValuesEquals(t *testing.T, a, b []*types.Value) {
assert.Equal(t, len(a), len(b))

for idx, left := range a {
assert.Truef(t, proto.Equal(left, b[idx]),
"Arrays are not equal. Diff[%d] %v != %v", idx, left, b[idx])
}
}