Skip to content

Commit b1efc80

Browse files
pyalexpyalex
authored
[Java feature server] Converge ServingService API to make Python and Java feature servers consistent (#2166)
* hgetall Signed-off-by: pyalex <moskalenko.alexey@gmail.com> optimized version of Serving proto Signed-off-by: pyalex <moskalenko.alexey@gmail.com> temp Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * refactored online service Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * java tests pass Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * remove project from request & entities from response Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * go sdk updated to use new protos Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * benchmark serving in ITs Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fix api docs build Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fixes after rebase Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * rename FeatureReferenceV2.name -> FeatureReferenceV2.feature_name Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * change proto property name in go sdk Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * refactoring FeastClient Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * add some comments Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * populate metrics Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * format after rebase Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * comment Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * todo added Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
1 parent d447db2 commit b1efc80

File tree

62 files changed

+2036
-1629
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+2036
-1629
lines changed

java/common/src/main/java/feast/common/models/FeatureV2.java renamed to java/common/src/main/java/feast/common/models/Feature.java

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
2020

21-
public class FeatureV2 {
21+
public class Feature {
2222

2323
/**
2424
* Accepts FeatureReferenceV2 object and returns its reference in String
@@ -27,10 +27,10 @@ public class FeatureV2 {
2727
* @param featureReference {@link FeatureReferenceV2}
2828
* @return String format of FeatureReferenceV2
2929
*/
30-
public static String getFeatureStringRef(FeatureReferenceV2 featureReference) {
31-
String ref = featureReference.getName();
32-
if (!featureReference.getFeatureTable().isEmpty()) {
33-
ref = featureReference.getFeatureTable() + ":" + ref;
30+
public static String getFeatureReference(FeatureReferenceV2 featureReference) {
31+
String ref = featureReference.getFeatureName();
32+
if (!featureReference.getFeatureViewName().isEmpty()) {
33+
ref = featureReference.getFeatureViewName() + ":" + ref;
3434
}
3535
return ref;
3636
}
@@ -47,4 +47,12 @@ public static String getFeatureName(String featureReference) {
4747
String[] tokens = featureReference.split(":", 2);
4848
return tokens[tokens.length - 1];
4949
}
50+
51+
public static FeatureReferenceV2 parseFeatureReference(String featureReference) {
52+
String[] tokens = featureReference.split(":", 2);
53+
return FeatureReferenceV2.newBuilder()
54+
.setFeatureViewName(tokens[0])
55+
.setFeatureName(tokens[1])
56+
.build();
57+
}
5058
}

‎java/common/src/main/java/feast/common/models/FeatureTable.java‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@ public List<AuditLogEntry> getTestAuditLogs() {
3939
.addAllFeatures(
4040
Arrays.asList(
4141
FeatureReferenceV2.newBuilder()
42-
.setFeatureTable("featuretable_1")
43-
.setName("feature1")
42+
.setFeatureViewName("featuretable_1")
43+
.setFeatureName("feature1")
4444
.build(),
4545
FeatureReferenceV2.newBuilder()
46-
.setFeatureTable("featuretable_1")
47-
.setName("feature2")
46+
.setFeatureViewName("featuretable_1")
47+
.setFeatureName("feature2")
4848
.build()))
4949
.build();
5050

‎java/common/src/test/java/feast/common/models/FeaturesTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@ public class FeaturesTest {
3131
public void setUp() {
3232
featureReference =
3333
FeatureReferenceV2.newBuilder()
34-
.setFeatureTable("featuretable_1")
35-
.setName("feature1")
34+
.setFeatureViewName("featuretable_1")
35+
.setFeatureName("feature1")
3636
.build();
3737
}
3838

3939
@Test
4040
public void shouldReturnFeatureStringRef() {
41-
String actualFeatureStringRef = FeatureV2.getFeatureStringRef(featureReference);
41+
String actualFeatureStringRef = Feature.getFeatureReference(featureReference);
4242
String expectedFeatureStringRef = "featuretable_1:feature1";
4343

4444
assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef));

java/sdk/java/src/main/java/com/gojek/feast/FeastClient.java renamed to java/sdk/java/src/main/java/dev/feast/FeastClient.java

Lines changed: 61 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,17 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package com.gojek.feast;
17+
package dev.feast;
1818

19-
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
19+
import com.google.common.collect.Lists;
20+
import feast.proto.serving.ServingAPIProto;
2021
import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
2122
import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse;
22-
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
23-
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow;
24-
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
23+
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest;
24+
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2;
2525
import feast.proto.serving.ServingServiceGrpc;
2626
import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub;
27+
import feast.proto.types.ValueProto;
2728
import io.grpc.CallCredentials;
2829
import io.grpc.ManagedChannel;
2930
import io.grpc.ManagedChannelBuilder;
@@ -32,9 +33,8 @@
3233
import io.opentracing.contrib.grpc.TracingClientInterceptor;
3334
import io.opentracing.util.GlobalTracer;
3435
import java.io.File;
35-
import java.util.HashSet;
36-
import java.util.List;
37-
import java.util.Optional;
36+
import java.time.Instant;
37+
import java.util.*;
3838
import java.util.concurrent.TimeUnit;
3939
import java.util.stream.Collectors;
4040
import javax.net.ssl.SSLException;
@@ -118,11 +118,60 @@ public GetFeastServingInfoResponse getFeastServingInfo() {
118118
* @param featureRefs list of string feature references to retrieve in the following format
119119
* featureTable:feature, where 'featureTable' and 'feature' refer to the FeatureTable and
120120
* Feature names respectively. Only the Feature name is required.
121-
* @param rows list of {@link Row} to select the entities to retrieve the features for.
121+
* @param entities list of {@link Row} to select the entities to retrieve the features for.
122122
* @return list of {@link Row} containing retrieved data fields.
123123
*/
124-
public List<Row> getOnlineFeatures(List<String> featureRefs, List<Row> rows) {
125-
return getOnlineFeatures(featureRefs, rows, "");
124+
public List<Row> getOnlineFeatures(List<String> featureRefs, List<Row> entities) {
125+
GetOnlineFeaturesRequest.Builder requestBuilder = GetOnlineFeaturesRequest.newBuilder();
126+
127+
requestBuilder.setFeatures(
128+
ServingAPIProto.FeatureList.newBuilder().addAllVal(featureRefs).build());
129+
130+
requestBuilder.putAllEntities(getEntityValuesMap(entities));
131+
132+
GetOnlineFeaturesResponseV2 response = stub.getOnlineFeatures(requestBuilder.build());
133+
134+
List<Row> results = Lists.newArrayList();
135+
if (response.getResultsCount() == 0) {
136+
return results;
137+
}
138+
139+
for (int rowIdx = 0; rowIdx < response.getResults(0).getValuesCount(); rowIdx++) {
140+
Row row = Row.create();
141+
for (int featureIdx = 0; featureIdx < response.getResultsCount(); featureIdx++) {
142+
row.set(
143+
response.getMetadata().getFeatureNames().getVal(featureIdx),
144+
response.getResults(featureIdx).getValues(rowIdx),
145+
response.getResults(featureIdx).getStatuses(rowIdx));
146+
147+
row.setEntityTimestamp(
148+
Instant.ofEpochSecond(
149+
response.getResults(featureIdx).getEventTimestamps(rowIdx).getSeconds()));
150+
}
151+
for (Map.Entry<String, ValueProto.Value> entry :
152+
entities.get(rowIdx).getFields().entrySet()) {
153+
row.set(entry.getKey(), entry.getValue());
154+
}
155+
156+
results.add(row);
157+
}
158+
return results;
159+
}
160+
161+
private Map<String, ValueProto.RepeatedValue> getEntityValuesMap(List<Row> entities) {
162+
Map<String, ValueProto.RepeatedValue.Builder> columnarEntities = new HashMap<>();
163+
for (Row row : entities) {
164+
for (Map.Entry<String, ValueProto.Value> field : row.getFields().entrySet()) {
165+
if (!columnarEntities.containsKey(field.getKey())) {
166+
columnarEntities.put(field.getKey(), ValueProto.RepeatedValue.newBuilder());
167+
}
168+
columnarEntities.get(field.getKey()).addVal(field.getValue());
169+
}
170+
}
171+
172+
return columnarEntities.entrySet().stream()
173+
.map((e) -> Map.entry(e.getKey(), e.getValue().build()))
174+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
126175
}
127176

128177
/**
@@ -149,42 +198,7 @@ public List<Row> getOnlineFeatures(List<String> featureRefs, List<Row> rows) {
149198
* @return list of {@link Row} containing retrieved data fields.
150199
*/
151200
public List<Row> getOnlineFeatures(List<String> featureRefs, List<Row> rows, String project) {
152-
List<FeatureReferenceV2> features = RequestUtil.createFeatureRefs(featureRefs);
153-
// build entity rows and collect entity references
154-
HashSet<String> entityRefs = new HashSet<>();
155-
List<EntityRow> entityRows =
156-
rows.stream()
157-
.map(
158-
row -> {
159-
entityRefs.addAll(row.getFields().keySet());
160-
return EntityRow.newBuilder()
161-
.setTimestamp(row.getEntityTimestamp())
162-
.putAllFields(row.getFields())
163-
.build();
164-
})
165-
.collect(Collectors.toList());
166-
167-
GetOnlineFeaturesResponse response =
168-
stub.getOnlineFeaturesV2(
169-
GetOnlineFeaturesRequestV2.newBuilder()
170-
.addAllFeatures(features)
171-
.addAllEntityRows(entityRows)
172-
.setProject(project)
173-
.build());
174-
175-
return response.getFieldValuesList().stream()
176-
.map(
177-
fieldValues -> {
178-
Row row = Row.create();
179-
for (String fieldName : fieldValues.getFieldsMap().keySet()) {
180-
row.set(
181-
fieldName,
182-
fieldValues.getFieldsMap().get(fieldName),
183-
fieldValues.getStatusesMap().get(fieldName));
184-
}
185-
return row;
186-
})
187-
.collect(Collectors.toList());
201+
return getOnlineFeatures(featureRefs, rows);
188202
}
189203

190204
protected FeastClient(ManagedChannel channel, Optional<CallCredentials> credentials) {

java/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java renamed to java/sdk/java/src/main/java/dev/feast/RequestUtil.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package com.gojek.feast;
17+
package dev.feast;
1818

1919
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
2020
import java.util.List;
@@ -71,8 +71,8 @@ public static FeatureReferenceV2 parseFeatureRef(String featureRefString) {
7171
String[] featureReferenceParts = featureRefString.split(":");
7272
FeatureReferenceV2 featureRef =
7373
FeatureReferenceV2.newBuilder()
74-
.setFeatureTable(featureReferenceParts[0])
75-
.setName(featureReferenceParts[1])
74+
.setFeatureViewName(featureReferenceParts[0])
75+
.setFeatureName(featureReferenceParts[1])
7676
.build();
7777

7878
return featureRef;

java/sdk/java/src/main/java/com/gojek/feast/Row.java renamed to java/sdk/java/src/main/java/dev/feast/Row.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package com.gojek.feast;
17+
package dev.feast;
1818

1919
import com.google.protobuf.ByteString;
2020
import com.google.protobuf.Timestamp;
2121
import com.google.protobuf.util.Timestamps;
22-
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus;
22+
import feast.proto.serving.ServingAPIProto.FieldStatus;
2323
import feast.proto.types.ValueProto.Value;
2424
import feast.proto.types.ValueProto.Value.ValCase;
2525
import java.time.Instant;

java/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java renamed to java/sdk/java/src/main/java/dev/feast/SecurityConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package com.gojek.feast;
17+
package dev.feast;
1818

1919
import com.google.auto.value.AutoValue;
2020
import io.grpc.CallCredentials;

0 commit comments

Comments
 (0)