getTestAuditLogs() {
+ GetOnlineFeaturesRequestV2 requestSpec =
+ GetOnlineFeaturesRequestV2.newBuilder()
+ .addAllFeatures(
+ Arrays.asList(
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature1")
+ .build(),
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature2")
+ .build()))
+ .build();
+
+ GetOnlineFeaturesResponse responseSpec =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addAllFieldValues(
+ Arrays.asList(
+ FieldValues.newBuilder()
+ .putFields(
+ "featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build())
+ .build(),
+ FieldValues.newBuilder()
+ .putFields(
+ "featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build())
+ .build()))
+ .build();
+
+ return Arrays.asList(
+ MessageAuditLogEntry.newBuilder()
+ .setComponent("feast-serving")
+ .setVersion("0.9")
+ .setService("ServingService")
+ .setMethod("getOnlineFeatures")
+ .setRequest(requestSpec)
+ .setResponse(responseSpec)
+ .setStatusCode(Status.OK.getCode())
+ .setIdentity("adam@no.such.email")
+ .build(),
+ ActionAuditLogEntry.of(
+ "core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
+ TransitionAuditLogEntry.of(
+ "core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY"));
+ }
+
+ @Test
+ public void shouldReturnJSONRepresentationOfAuditLog() {
+ for (AuditLogEntry auditLog : getTestAuditLogs()) {
+ // Check that auditLog's toJSON() returns valid JSON
+ String logJSON = auditLog.toJSON();
+ System.out.println(logJSON);
+ JsonParser parser = new JsonParser();
+
+ // check basic fields are present in JSON representation.
+ JsonObject logObject = parser.parse(logJSON).getAsJsonObject();
+ assertThat(logObject.getAsJsonPrimitive("logType").getAsString(), equalTo(auditLog.logType));
+ assertThat(
+ logObject.getAsJsonPrimitive("kind").getAsString(), equalTo(auditLog.getKind().name()));
+ }
+ }
+}
diff --git a/java/common/src/test/java/feast/common/models/FeaturesTest.java b/java/common/src/test/java/feast/common/models/FeaturesTest.java
new file mode 100644
index 00000000000..180f7e4e697
--- /dev/null
+++ b/java/common/src/test/java/feast/common/models/FeaturesTest.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.common.models;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FeaturesTest {
+
+ private FeatureReferenceV2 featureReference;
+
+ @Before
+ public void setUp() {
+ featureReference =
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature1")
+ .build();
+ }
+
+ @Test
+ public void shouldReturnFeatureStringRef() {
+ String actualFeatureStringRef = FeatureV2.getFeatureStringRef(featureReference);
+ String expectedFeatureStringRef = "featuretable_1:feature1";
+
+ assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef));
+ }
+}
diff --git a/java/datatypes/java/README.md b/java/datatypes/java/README.md
new file mode 100644
index 00000000000..7fc355f7739
--- /dev/null
+++ b/java/datatypes/java/README.md
@@ -0,0 +1,55 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+
+ dev.feast
+ datatypes-java
+ 0.26.2
+
+```
+
+Use the version corresponding to the Feast release you have deployed in your
+environment—see the [Feast release notes] for details.
+
+[Feast release notes]: ../../CHANGELOG.md
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Releases
+--------
+
+The module is published to Maven Central upon each release of Feast (since
+v0.3.7).
+
+For developers, the publishing process is automated along with the Java SDK by
+[the `publish-java-sdk` build task in Prow][prow task], where you can see how
+it works. Artifacts are staged to Sonatype where a maintainer needs to take a
+release action for them to go live on Maven Central.
+
+[prow task]: https://github.com/feast-dev/feast/blob/17e7dca8238aae4dcbf0ff9f0db5d80ef8e035cf/.prow/config.yaml#L166-L192
diff --git a/java/datatypes/java/pom.xml b/java/datatypes/java/pom.xml
new file mode 100644
index 00000000000..0d165f0cd53
--- /dev/null
+++ b/java/datatypes/java/pom.xml
@@ -0,0 +1,111 @@
+
+
+
+ 4.0.0
+
+ Feast Data Types for Java
+
+ Data types and service contracts used throughout Feast components and
+ their interchanges. These are generated from Protocol Buffers and gRPC
+ definitions included in the package.
+
+ datatypes-java
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../..
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+
+ javax.annotation
+
+
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+
+ true
+
+ com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
+
+ grpc-java
+
+ io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
+
+
+
+
+
+ compile
+ compile-custom
+ test-compile
+
+
+
+
+
+
+
+
+
+
+ com.google.guava
+ guava
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+
+ io.grpc
+ grpc-core
+
+
+ io.grpc
+ grpc-protobuf
+
+
+ io.grpc
+ grpc-services
+
+
+ io.grpc
+ grpc-stub
+
+
+
+ javax.annotation
+ javax.annotation-api
+
+
+
+
diff --git a/java/datatypes/java/src/main/proto/feast b/java/datatypes/java/src/main/proto/feast
new file mode 120000
index 00000000000..53364e5f45f
--- /dev/null
+++ b/java/datatypes/java/src/main/proto/feast
@@ -0,0 +1 @@
+../../../../../../protos/feast
\ No newline at end of file
diff --git a/java/datatypes/java/src/main/proto/tensorflow_metadata b/java/datatypes/java/src/main/proto/tensorflow_metadata
new file mode 120000
index 00000000000..225345970c2
--- /dev/null
+++ b/java/datatypes/java/src/main/proto/tensorflow_metadata
@@ -0,0 +1 @@
+../../../../../../protos/tensorflow_metadata
\ No newline at end of file
diff --git a/java/docs/coverage/java/pom.xml b/java/docs/coverage/java/pom.xml
new file mode 100644
index 00000000000..5f794224969
--- /dev/null
+++ b/java/docs/coverage/java/pom.xml
@@ -0,0 +1,87 @@
+
+
+
+ 4.0.0
+
+
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../../..
+
+
+ Feast Coverage Java
+ feast-coverage
+
+
+ true
+
+
+
+
+ dev.feast
+ feast-storage-api
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-redis
+ ${project.version}
+
+
+
+ dev.feast
+ feast-serving
+ ${project.version}
+
+
+
+ dev.feast
+ feast-sdk
+ ${project.version}
+
+
+
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+ report-aggregate
+ prepare-package
+
+ report-aggregate
+
+
+
+
+
+
+
+
diff --git a/java/infra/docker/serving/Dockerfile b/java/infra/docker/serving/Dockerfile
new file mode 100644
index 00000000000..0aacf856305
--- /dev/null
+++ b/java/infra/docker/serving/Dockerfile
@@ -0,0 +1,53 @@
+# ============================================================
+# Build stage 1: Builder
+# ============================================================
+
+FROM maven:3.6-jdk-11 as builder
+
+WORKDIR /build
+
+COPY pom.xml .
+COPY datatypes/java/pom.xml datatypes/java/pom.xml
+COPY common/pom.xml common/pom.xml
+COPY core/pom.xml core/pom.xml
+COPY serving/pom.xml serving/pom.xml
+COPY storage/api/pom.xml storage/api/pom.xml
+COPY storage/connectors/pom.xml storage/connectors/pom.xml
+COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml
+COPY sdk/java/pom.xml sdk/java/pom.xml
+COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml
+COPY deps/feast/protos/ deps/feast/protos/
+
+# Setting Maven repository .m2 directory relative to /build folder gives the
+# user to optionally use cached repository when building the image by copying
+# the existing .m2 directory to $FEAST_REPO_ROOT/.m2
+ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3"
+COPY pom.xml .m2/* .m2/
+RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true
+
+COPY . .
+
+ARG VERSION=dev
+RUN mvn --also-make --projects serving -Drevision=$VERSION \
+ -DskipUTs=true --batch-mode clean package
+#
+# Download grpc_health_probe to run health check for Feast Serving
+# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/
+#
+RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \
+ -O /usr/bin/grpc-health-probe && \
+ chmod +x /usr/bin/grpc-health-probe
+
+# ============================================================
+# Build stage 2: Production
+# ============================================================
+
+FROM amazoncorretto:11 as production
+ARG VERSION=dev
+COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar
+COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe
+CMD ["java",\
+ "-Xms1g",\
+ "-Xmx4g",\
+ "-jar",\
+ "/opt/feast/feast-serving.jar"]
diff --git a/java/infra/docker/serving/Dockerfile.dev b/java/infra/docker/serving/Dockerfile.dev
new file mode 100644
index 00000000000..93bbbbb7185
--- /dev/null
+++ b/java/infra/docker/serving/Dockerfile.dev
@@ -0,0 +1,15 @@
+FROM openjdk:11-jre as production
+ARG REVISION=dev
+#
+# Download grpc_health_probe to run health check for Feast Serving
+# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/
+#
+RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \
+ -O /usr/bin/grpc-health-probe && \
+ chmod +x /usr/bin/grpc-health-probe
+ADD $PWD/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar
+CMD ["java",\
+ "-Xms1024m",\
+ "-Xmx1024m",\
+ "-jar",\
+ "/opt/feast/feast-serving.jar"]
diff --git a/java/pom.xml b/java/pom.xml
new file mode 100644
index 00000000000..073616b69a4
--- /dev/null
+++ b/java/pom.xml
@@ -0,0 +1,855 @@
+
+
+
+ 4.0.0
+
+ Feast
+ Feature Store for Machine Learning
+ ${github.url}
+
+ dev.feast
+ feast-parent
+ ${revision}
+ pom
+
+
+ datatypes/java
+ storage/api
+ storage/connectors
+ serving
+ sdk/java
+ docs/coverage/java
+ common
+
+
+
+ 0.26.2
+ https://github.com/feast-dev/feast
+
+ UTF-8
+ UTF-8
+
+ 1.30.2
+ 3.12.2
+ 3.12.2
+ 2.3.1.RELEASE
+ 5.2.7.RELEASE
+ 5.3.0.RELEASE
+ 2.9.0.RELEASE
+ 1.111.1
+ 0.8.0
+ 1.9.10
+ 1.3
+ 2.28.2
+
+ 0.26.0
+
+ 2.13.2
+ 2.9.9
+ 2.0.2
+ 2.5.0.RELEASE
+ 1.18.12
+ 1.8.4
+ 2.8.6
+ 1.5.24
+ 3.14.7
+ 3.10
+ 2.3.1
+ 1.3.2
+ 2.0.1.Final
+ 2.8.0
+ 0.20.0
+ 1.6.6
+
+
+
+ ${maven.multiModuleProjectDirectory}
+
+ false
+ false
+ feast.common.auth.providers.http.client
+
+
+
+ Gojek
+ https://www.gojek.com
+
+
+
+
+ Feast Authors
+ ${github.url}
+ Gojek
+ https://www.gojek.com
+
+
+
+
+
+ Apache License, Version 2.0
+ https://www.apache.org/licenses/LICENSE-2.0.txt
+ repo
+
+
+
+
+ ${github.url}
+ scm:git:${github.url}.git
+ scm:git:git@github.com:feast-dev/feast.git
+ HEAD
+
+
+
+ GitHub Issues
+ ${github.url}/issues
+
+
+
+
+
+ ossrh
+ https://oss.sonatype.org/content/repositories/snapshots
+
+
+ ossrh
+ https://oss.sonatype.org/service/local/staging/deploy/maven2/
+
+
+
+
+
+
+ org.apache.commons
+ commons-lang3
+ ${commons.lang3.version}
+
+
+
+
+ com.google.cloud
+ google-cloud-bigquery
+ ${com.google.cloud.version}
+
+
+ com.google.cloud
+ google-cloud-storage
+ ${com.google.cloud.version}
+
+
+
+
+ com.google.cloud
+ google-cloud-nio
+ 0.83.0-alpha
+
+
+
+ io.opencensus
+ opencensus-api
+ ${opencensus.version}
+
+
+ io.opencensus
+ opencensus-contrib-grpc-util
+ ${opencensus.version}
+
+
+ io.opencensus
+ opencensus-contrib-http-util
+ ${opencensus.version}
+
+
+
+
+ io.grpc
+ grpc-core
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-api
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-context
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-all
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-okhttp
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-auth
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-grpclb
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-alts
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-netty
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-netty-shaded
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-protobuf
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-services
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-stub
+ ${grpc.version}
+
+
+ io.grpc
+ grpc-testing
+ ${grpc.version}
+ test
+
+
+
+
+ org.apache.arrow
+ arrow-java-root
+ 5.0.0
+ pom
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+ 5.0.0
+
+
+
+
+ org.apache.arrow
+ arrow-memory
+ 5.0.0
+ pom
+
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+ 5.0.0
+ runtime
+
+
+
+
+ net.devh
+ grpc-server-spring-boot-starter
+ ${grpc.spring.boot.starter.version}
+
+
+
+
+ io.prometheus
+ simpleclient
+ ${io.prometheus.version}
+
+
+ io.prometheus
+ simpleclient_servlet
+ ${io.prometheus.version}
+
+
+
+
+ org.springframework.security
+ spring-security-core
+ ${spring.security.version}
+
+
+ org.springframework.security
+ spring-security-config
+ ${spring.security.version}
+
+
+ org.springframework.security
+ spring-security-oauth2-resource-server
+ ${spring.security.version}
+
+
+ org.springframework.security
+ spring-security-oauth2-jose
+ ${spring.security.version}
+
+
+ com.google.auth
+ google-auth-library-oauth2-http
+ ${google.auth.library.oauth2.http.version}
+
+
+
+
+ joda-time
+ joda-time
+ ${joda.time.version}
+
+
+ com.datadoghq
+ java-dogstatsd-client
+ 2.6.1
+
+
+ com.google.guava
+ guava
+ 29.0-jre
+
+
+ com.google.protobuf
+ protobuf-java
+ ${protobuf.version}
+
+
+ com.google.protobuf
+ protobuf-java-util
+ ${protobuf.version}
+
+
+ org.projectlombok
+ lombok
+ ${lombok.version}
+ provided
+
+
+ com.google.auto.value
+ auto-value-annotations
+ ${auto.value.version}
+
+
+ com.google.auto.value
+ auto-value
+ ${auto.value.version}
+
+
+ com.google.code.gson
+ gson
+ ${gson.version}
+
+
+ io.gsonfire
+ gson-fire
+ ${gson.fire.version}
+
+
+
+ com.github.kstyrc
+ embedded-redis
+ 0.6
+ test
+
+
+
+
+
+ net.bytebuddy
+ byte-buddy
+ ${byte-buddy.version}
+
+
+ org.mockito
+ mockito-core
+ ${mockito.version}
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+ ${spring.boot.version}
+
+
+ org.springframework.boot
+ spring-boot-starter-logging
+
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${log4jVersion}
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${log4jVersion}
+
+
+ org.apache.logging.log4j
+ log4j-jul
+ ${log4jVersion}
+
+
+ org.apache.logging.log4j
+ log4j-web
+ ${log4jVersion}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ ${log4jVersion}
+
+
+
+
+ org.springframework.boot
+ spring-boot-dependencies
+ ${spring.boot.version}
+ pom
+ import
+
+
+ com.squareup.okio
+ okio
+ 1.17.2
+
+
+ javax.xml.bind
+ jaxb-api
+ ${javax.xml.bind.version}
+
+
+ javax.annotation
+ javax.annotation-api
+ ${javax.annotation.version}
+
+
+ javax.validation
+ validation-api
+ ${javax.validation.version}
+
+
+
+
+
+
+
+ kr.motd.maven
+ os-maven-plugin
+ 1.6.2
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.2.1
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
+ feast.proto.*:io.grpc.*:org.tensorflow.*
+
+
+
+ com.diffplug.spotless
+ spotless-maven-plugin
+ 1.26.1
+
+
+
+ ${license.content}
+
+
+ 1.7
+
+
+
+ src/main/java/**/BatchLoadsWithResult.java
+
+
+
+
+
+ ${license.content}
+
+
+
+
+
+
+ spotless-check
+ process-test-classes
+
+ check
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+
+ 8
+
+
+ com.google.auto.value
+ auto-value
+ ${auto.value.version}
+
+
+ org.projectlombok
+ lombok
+ ${lombok.version}
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-enforcer-plugin
+ 3.0.0-M2
+
+
+ org.codehaus.mojo
+ extra-enforcer-rules
+ 1.2
+
+
+
+
+ valid-build-environment
+
+ enforce
+
+
+
+
+ [3.6,4.0)
+
+
+ [11.0,)
+
+
+
+
+
+
+ consistent-dependency-versions
+
+ enforce
+
+
+
+
+
+
+
+
+ no-snapshot-deps-at-release
+
+ enforce
+
+
+
+
+ true
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 3.0.0-M5
+
+ @{argLine} -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true
+ ${skipUTs}
+
+
+
+ org.junit.vintage
+ junit-vintage-engine
+ 5.5.2
+
+
+
+
+ maven-failsafe-plugin
+ 3.0.0-M5
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ 5.6.2
+
+
+
+
+ integration-tests
+
+ integration-test
+ verify
+
+
+
+
+
+ ${project.groupId}:${project.artifactId}
+
+
+ ${project.build.outputDirectory}
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+
+ true
+
+
+
+ build-info
+
+ build-info
+
+
+
+
+
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.8
+ true
+
+ ossrh
+ https://oss.sonatype.org/
+
+ true
+
+
+
+
+ org.codehaus.mojo
+ flatten-maven-plugin
+ 1.1.0
+
+ oss
+
+
+
+ flatten
+ process-resources
+
+ flatten
+
+
+
+ flatten.clean
+ clean
+
+ clean
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+ 1.6
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+
+
+ --pinentry-mode
+ loopback
+
+
+
+ ${gpg.passphrase}
+
+
+
+
+
+
+
+
+
+ io.fabric8
+ docker-maven-plugin
+ 0.20.1
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.8.1
+
+
+ -Xlint:all
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.1.1
+
+
+
+ org.apache.maven.shared
+ maven-dependency-analyzer
+ 1.11.1
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.1.1
+
+ all
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ 1.6.0
+
+ false
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.8.5
+
+
+
+ prepare-agent
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+ ${spring.boot.version}
+
+
+
+ repackage
+
+
+
+
+ exec
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+ 0.6.1
+
+
+
+
+
diff --git a/java/sdk/java/pom.xml b/java/sdk/java/pom.xml
new file mode 100644
index 00000000000..6b2295b6b33
--- /dev/null
+++ b/java/sdk/java/pom.xml
@@ -0,0 +1,143 @@
+
+
+ 4.0.0
+
+ Feast SDK for Java
+ SDK for registering, storing, and retrieving features
+ feast-sdk
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../..
+
+
+
+
+ 5.5.2
+ 2.28.2
+ 0.33.0
+
+
+
+
+ dev.feast
+ datatypes-java
+ ${project.version}
+
+
+
+ dev.feast
+ feast-common
+ ${project.version}
+
+
+
+
+ io.grpc
+ grpc-netty-shaded
+
+
+ io.grpc
+ grpc-protobuf
+
+
+ io.grpc
+ grpc-stub
+
+
+ io.grpc
+ grpc-testing
+
+
+ com.google.protobuf
+ protobuf-java-util
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+
+
+ io.opentracing.contrib
+ opentracing-grpc
+ 0.2.3
+
+
+ io.opentracing
+ opentracing-api
+ ${opentracing.version}
+
+
+ io.opentracing
+ opentracing-noop
+ ${opentracing.version}
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ ${junit.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ ${junit.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ ${junit.version}
+ test
+
+
+ org.apache.commons
+ commons-lang3
+ 3.6
+ compile
+
+
+ org.mockito
+ mockito-core
+ ${mockito.version}
+ test
+
+
+ org.mockito
+ mockito-inline
+ ${mockito.version}
+ test
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+ true
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+
+
diff --git a/java/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/java/sdk/java/src/main/java/com/gojek/feast/FeastClient.java
new file mode 100644
index 00000000000..94836d88aa4
--- /dev/null
+++ b/java/sdk/java/src/main/java/com/gojek/feast/FeastClient.java
@@ -0,0 +1,211 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingServiceGrpc;
+import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub;
+import io.grpc.CallCredentials;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.opentracing.contrib.grpc.TracingClientInterceptor;
+import io.opentracing.util.GlobalTracer;
+import java.io.File;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import javax.net.ssl.SSLException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("WeakerAccess")
+public class FeastClient implements AutoCloseable {
+ Logger logger = LoggerFactory.getLogger(FeastClient.class);
+
+ private static final int CHANNEL_SHUTDOWN_TIMEOUT_SEC = 5;
+
+ private final ManagedChannel channel;
+ private final ServingServiceBlockingStub stub;
+
+ /**
+ * Create a client to access Feast Serving.
+ *
+ * @param host hostname or ip address of Feast serving GRPC server
+ * @param port port number of Feast serving GRPC server
+ * @return {@link FeastClient}
+ */
+ public static FeastClient create(String host, int port) {
+ // configure client with no security config.
+ return FeastClient.createSecure(host, port, SecurityConfig.newBuilder().build());
+ }
+
+ /**
+ * Create a authenticated client that can access Feast serving with authentication enabled.
+ * Supports the {@link CallCredentials} in the {@link feast.common.auth.credentials} package.
+ *
+ * @param host hostname or ip address of Feast serving GRPC server
+ * @param port port number of Feast serving GRPC server
+ * @param securityConfig security options to configure the Feast client. See {@link
+ * SecurityConfig} for options.
+ * @return {@link FeastClient}
+ */
+ public static FeastClient createSecure(String host, int port, SecurityConfig securityConfig) {
+ // Configure client TLS
+ ManagedChannel channel = null;
+ if (securityConfig.isTLSEnabled()) {
+ if (securityConfig.getCertificatePath().isPresent()) {
+ String certificatePath = securityConfig.getCertificatePath().get();
+ // Use custom certificate for TLS
+ File certificateFile = new File(certificatePath);
+ try {
+ channel =
+ NettyChannelBuilder.forAddress(host, port)
+ .useTransportSecurity()
+ .sslContext(GrpcSslContexts.forClient().trustManager(certificateFile).build())
+ .build();
+ } catch (SSLException e) {
+ throw new IllegalArgumentException(
+ String.format("Invalid Certificate provided at path: %s", certificatePath), e);
+ }
+ } else {
+ // Use system certificates for TLS
+ channel = ManagedChannelBuilder.forAddress(host, port).useTransportSecurity().build();
+ }
+ } else {
+ // Disable TLS
+ channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build();
+ }
+
+ return new FeastClient(channel, securityConfig.getCredentials());
+ }
+
+ /**
+ * Obtain info about Feast Serving.
+ *
+ * @return {@link GetFeastServingInfoResponse} containing Feast version, Serving type etc.
+ */
+ public GetFeastServingInfoResponse getFeastServingInfo() {
+ return stub.getFeastServingInfo(GetFeastServingInfoRequest.newBuilder().build());
+ }
+
+ /**
+ * Get online features from Feast, without indicating project, will use `default`.
+ *
+ * See {@link #getOnlineFeatures(List, List, String)}
+ *
+ * @param featureRefs list of string feature references to retrieve in the following format
+ * featureTable:feature, where 'featureTable' and 'feature' refer to the FeatureTable and
+ * Feature names respectively. Only the Feature name is required.
+ * @param rows list of {@link Row} to select the entities to retrieve the features for.
+ * @return list of {@link Row} containing retrieved data fields.
+ */
+ public List getOnlineFeatures(List featureRefs, List rows) {
+ return getOnlineFeatures(featureRefs, rows, "");
+ }
+
+ /**
+ * Get online features from Feast.
+ *
+ * Example of retrieving online features for the driver FeatureTable, with features driver_id
+ * and driver_name
+ *
+ *
{@code
+ * FeastClient client = FeastClient.create("localhost", 6566);
+ * List requestedFeatureIds = Arrays.asList("driver:driver_id", "driver:driver_name");
+ * List requestedRows =
+ * Arrays.asList(Row.create().set("driver_id", 123), Row.create().set("driver_id", 456));
+ * List retrievedFeatures = client.getOnlineFeatures(requestedFeatureIds, requestedRows);
+ * retrievedFeatures.forEach(System.out::println);
+ * }
+ *
+ * @param featureRefs list of string feature references to retrieve in the following format
+ * featureTable:feature, where 'featureTable' and 'feature' refer to the FeatureTable and
+ * Feature names respectively. Only the Feature name is required.
+ * @param rows list of {@link Row} to select the entities to retrieve the features for
+ * @param project {@link String} Specifies the project override. If specified uses the project for
+ * retrieval. Overrides the projects set in Feature References if also specified.
+ * @return list of {@link Row} containing retrieved data fields.
+ */
+ public List getOnlineFeatures(List featureRefs, List rows, String project) {
+ List features = RequestUtil.createFeatureRefs(featureRefs);
+ // build entity rows and collect entity references
+ HashSet entityRefs = new HashSet<>();
+ List entityRows =
+ rows.stream()
+ .map(
+ row -> {
+ entityRefs.addAll(row.getFields().keySet());
+ return EntityRow.newBuilder()
+ .setTimestamp(row.getEntityTimestamp())
+ .putAllFields(row.getFields())
+ .build();
+ })
+ .collect(Collectors.toList());
+
+ GetOnlineFeaturesResponse response =
+ stub.getOnlineFeaturesV2(
+ GetOnlineFeaturesRequestV2.newBuilder()
+ .addAllFeatures(features)
+ .addAllEntityRows(entityRows)
+ .setProject(project)
+ .build());
+
+ return response.getFieldValuesList().stream()
+ .map(
+ fieldValues -> {
+ Row row = Row.create();
+ for (String fieldName : fieldValues.getFieldsMap().keySet()) {
+ row.set(
+ fieldName,
+ fieldValues.getFieldsMap().get(fieldName),
+ fieldValues.getStatusesMap().get(fieldName));
+ }
+ return row;
+ })
+ .collect(Collectors.toList());
+ }
+
+ protected FeastClient(ManagedChannel channel, Optional credentials) {
+ this.channel = channel;
+ TracingClientInterceptor tracingInterceptor =
+ TracingClientInterceptor.newBuilder().withTracer(GlobalTracer.get()).build();
+
+ ServingServiceBlockingStub servingStub =
+ ServingServiceGrpc.newBlockingStub(tracingInterceptor.intercept(channel));
+
+ if (credentials.isPresent()) {
+ servingStub = servingStub.withCallCredentials(credentials.get());
+ }
+
+ this.stub = servingStub;
+ }
+
+ public void close() throws Exception {
+ if (channel != null) {
+ channel.shutdown().awaitTermination(CHANNEL_SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS);
+ }
+ }
+}
diff --git a/java/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java b/java/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java
new file mode 100644
index 00000000000..69c8f9f737a
--- /dev/null
+++ b/java/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@SuppressWarnings("WeakerAccess")
+public class RequestUtil {
+
+ /**
+ * Create feature references protos from given string feature reference.
+ *
+ * @param featureRefStrings to create Feature Reference protos from
+ * @return List of parsed {@link FeatureReferenceV2} protos
+ */
+ public static List createFeatureRefs(List featureRefStrings) {
+ if (featureRefStrings == null) {
+ throw new IllegalArgumentException("FeatureReferences cannot be null");
+ }
+
+ List featureRefs =
+ featureRefStrings.stream()
+ .map(refStr -> parseFeatureRef(refStr))
+ .collect(Collectors.toList());
+
+ return featureRefs;
+ }
+
+ /**
+ * Parse a feature reference proto builder from the given featureRefString
+ *
+ * @param featureRefString string feature reference to parse from.
+ * @return a parsed {@link FeatureReferenceV2}
+ */
+ public static FeatureReferenceV2 parseFeatureRef(String featureRefString) {
+ featureRefString = featureRefString.trim();
+ if (featureRefString.isEmpty()) {
+ throw new IllegalArgumentException("Cannot parse a empty feature reference");
+ }
+ if (featureRefString.contains("/")) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Unsupported feature reference: Specifying project in string"
+ + " Feature References is not longer supported: %s",
+ featureRefString));
+ }
+ if (!featureRefString.contains(":")) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Unsupported feature reference: %s - FeatureTable name and Feature name should be provided in string"
+ + " Feature References, in : format.",
+ featureRefString));
+ }
+
+ String[] featureReferenceParts = featureRefString.split(":");
+ FeatureReferenceV2 featureRef =
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable(featureReferenceParts[0])
+ .setName(featureReferenceParts[1])
+ .build();
+
+ return featureRef;
+ }
+}
diff --git a/java/sdk/java/src/main/java/com/gojek/feast/Row.java b/java/sdk/java/src/main/java/com/gojek/feast/Row.java
new file mode 100644
index 00000000000..51f820e320a
--- /dev/null
+++ b/java/sdk/java/src/main/java/com/gojek/feast/Row.java
@@ -0,0 +1,161 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Timestamp;
+import com.google.protobuf.util.Timestamps;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus;
+import feast.proto.types.ValueProto.Value;
+import feast.proto.types.ValueProto.Value.ValCase;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+@SuppressWarnings("UnusedReturnValue")
+public class Row {
+ private Timestamp entity_timestamp;
+ private HashMap fields;
+ private HashMap fieldStatuses;
+
+ public static Row create() {
+ Row row = new Row();
+ row.entity_timestamp = Timestamps.fromMillis(System.currentTimeMillis());
+ row.fields = new HashMap<>();
+ row.fieldStatuses = new HashMap<>();
+ return row;
+ }
+
+ public Row setEntityTimestamp(Instant timestamp) {
+ entity_timestamp = Timestamps.fromMillis(timestamp.toEpochMilli());
+ return this;
+ }
+
+ public Timestamp getEntityTimestamp() {
+ return entity_timestamp;
+ }
+
+ public Row setEntityTimestamp(String dateTime) {
+ entity_timestamp = Timestamps.fromMillis(Instant.parse(dateTime).toEpochMilli());
+ return this;
+ }
+
+ public Row set(String fieldName, Object value) {
+ return this.set(fieldName, value, FieldStatus.PRESENT);
+ }
+
+ public Row set(String fieldName, Object value, FieldStatus status) {
+ String valueType = value.getClass().getCanonicalName();
+ switch (valueType) {
+ case "java.lang.Integer":
+ fields.put(fieldName, Value.newBuilder().setInt32Val((int) value).build());
+ break;
+ case "java.lang.Long":
+ fields.put(fieldName, Value.newBuilder().setInt64Val((long) value).build());
+ break;
+ case "java.lang.Float":
+ fields.put(fieldName, Value.newBuilder().setFloatVal((float) value).build());
+ break;
+ case "java.lang.Double":
+ fields.put(fieldName, Value.newBuilder().setDoubleVal((double) value).build());
+ break;
+ case "java.lang.String":
+ fields.put(fieldName, Value.newBuilder().setStringVal((String) value).build());
+ break;
+ case "byte[]":
+ fields.put(
+ fieldName, Value.newBuilder().setBytesVal(ByteString.copyFrom((byte[]) value)).build());
+ break;
+ case "feast.proto.types.ValueProto.Value":
+ fields.put(fieldName, (Value) value);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Type '%s' is unsupported in Feast. Please use one of these value types: Integer, Long, Float, Double, String, byte[].",
+ valueType));
+ }
+
+ fieldStatuses.put(fieldName, status);
+ return this;
+ }
+
+ public Map getFields() {
+ return fields;
+ }
+
+ public Integer getInt(String fieldName) {
+ return getValue(fieldName).map(Value::getInt32Val).orElse(null);
+ }
+
+ public Long getLong(String fieldName) {
+ return getValue(fieldName).map(Value::getInt64Val).orElse(null);
+ }
+
+ public Float getFloat(String fieldName) {
+ return getValue(fieldName).map(Value::getFloatVal).orElse(null);
+ }
+
+ public Double getDouble(String fieldName) {
+ return getValue(fieldName).map(Value::getDoubleVal).orElse(null);
+ }
+
+ public String getString(String fieldName) {
+ return getValue(fieldName).map(Value::getStringVal).orElse(null);
+ }
+
+ public byte[] getByte(String fieldName) {
+ return getValue(fieldName).map(Value::getBytesVal).map(ByteString::toByteArray).orElse(null);
+ }
+
+ public Map getStatuses() {
+ return fieldStatuses;
+ }
+
+ public FieldStatus getStatus(String fieldName) {
+ return fieldStatuses.get(fieldName);
+ }
+
+ @Override
+ public String toString() {
+ List parts = new ArrayList<>();
+ fields.forEach(
+ (key, value) ->
+ parts.add(
+ key
+ + ":"
+ + (value.getValCase().equals(ValCase.VAL_NOT_SET)
+ ? "NULL"
+ : value.toString().trim())));
+ return String.join(", ", parts);
+ }
+
+ private Optional getValue(String fieldName) {
+ if (!fields.containsKey(fieldName)) {
+ throw new IllegalArgumentException(
+ String.format("Row does not contain field '%s'", fieldName));
+ }
+ Value value = fields.get(fieldName);
+ if (value.getValCase().equals(ValCase.VAL_NOT_SET)) {
+ return Optional.empty();
+ }
+ return Optional.of(value);
+ }
+}
diff --git a/java/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java b/java/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java
new file mode 100644
index 00000000000..94c779cf440
--- /dev/null
+++ b/java/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import com.google.auto.value.AutoValue;
+import io.grpc.CallCredentials;
+import java.util.Optional;
+
+/** SecurityConfig captures the security related configuration for FeastClient */
+@AutoValue
+public abstract class SecurityConfig {
+ /**
+ * Enables authentication If specified, the call credentials used to provide credentials to
+ * authenticate with Feast.
+ *
+ * @return credentials
+ */
+ public abstract Optional getCredentials();
+
+ /**
+ * Whether to use TLS transport security is use when connecting to Feast.
+ *
+ * @return true if enabled
+ */
+ public abstract boolean isTLSEnabled();
+
+ /**
+ * If specified and TLS is enabled, provides path to TLS certificate use the verify Service
+ * identity.
+ *
+ * @return certificate path
+ */
+ public abstract Optional getCertificatePath();
+
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder setCredentials(Optional credentials);
+
+ public abstract Builder setTLSEnabled(boolean isTLSEnabled);
+
+ public abstract Builder setCertificatePath(Optional certificatePath);
+
+ public abstract SecurityConfig build();
+ }
+
+ public static SecurityConfig.Builder newBuilder() {
+ return new AutoValue_SecurityConfig.Builder()
+ .setCredentials(Optional.empty())
+ .setTLSEnabled(false)
+ .setCertificatePath(Optional.empty());
+ }
+}
diff --git a/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java
new file mode 100644
index 00000000000..29185cd153c
--- /dev/null
+++ b/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java
@@ -0,0 +1,168 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.AdditionalAnswers.delegatesTo;
+import static org.mockito.Mockito.mock;
+
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase;
+import feast.proto.types.ValueProto.Value;
+import io.grpc.*;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
+import io.grpc.stub.StreamObserver;
+import io.grpc.testing.GrpcCleanupRule;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class FeastClientTest {
+ private final String AUTH_TOKEN = "test token";
+
+ @Rule public GrpcCleanupRule grpcRule;
+ private AtomicBoolean isAuthenticated;
+
+ private ServingServiceImplBase servingMock =
+ mock(
+ ServingServiceImplBase.class,
+ delegatesTo(
+ new ServingServiceImplBase() {
+ @Override
+ public void getOnlineFeaturesV2(
+ GetOnlineFeaturesRequestV2 request,
+ StreamObserver responseObserver) {
+ if (!request.equals(FeastClientTest.getFakeRequest())) {
+ responseObserver.onError(Status.FAILED_PRECONDITION.asRuntimeException());
+ }
+
+ responseObserver.onNext(FeastClientTest.getFakeResponse());
+ responseObserver.onCompleted();
+ }
+ }));
+
+ private FeastClient client;
+
+ @Before
+ public void setup() throws Exception {
+ this.grpcRule = new GrpcCleanupRule();
+ this.isAuthenticated = new AtomicBoolean(false);
+ // setup fake serving service
+ String serverName = InProcessServerBuilder.generateName();
+ this.grpcRule.register(
+ InProcessServerBuilder.forName(serverName)
+ .directExecutor()
+ .addService(this.servingMock)
+ .build()
+ .start());
+
+ // setup test feast client target
+ ManagedChannel channel =
+ this.grpcRule.register(
+ InProcessChannelBuilder.forName(serverName).directExecutor().build());
+ this.client = new FeastClient(channel, Optional.empty());
+ }
+
+ @Test
+ public void shouldGetOnlineFeatures() {
+ shouldGetOnlineFeaturesWithClient(this.client);
+ }
+
+ private void shouldGetOnlineFeaturesWithClient(FeastClient client) {
+ List rows =
+ client.getOnlineFeatures(
+ Arrays.asList("driver:name", "driver:rating", "driver:null_value"),
+ Arrays.asList(
+ Row.create().set("driver_id", 1).setEntityTimestamp(Instant.ofEpochSecond(100))),
+ "driver_project");
+
+ assertEquals(
+ rows.get(0).getFields(),
+ new HashMap() {
+ {
+ put("driver_id", intValue(1));
+ put("driver:name", strValue("david"));
+ put("driver:rating", intValue(3));
+ put("driver:null_value", Value.newBuilder().build());
+ }
+ });
+ assertEquals(
+ rows.get(0).getStatuses(),
+ new HashMap() {
+ {
+ put("driver_id", FieldStatus.PRESENT);
+ put("driver:name", FieldStatus.PRESENT);
+ put("driver:rating", FieldStatus.PRESENT);
+ put("driver:null_value", FieldStatus.NULL_VALUE);
+ }
+ });
+ }
+
+ private static GetOnlineFeaturesRequestV2 getFakeRequest() {
+ // setup mock serving service stub
+ return GetOnlineFeaturesRequestV2.newBuilder()
+ .addFeatures(
+ FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("name").build())
+ .addFeatures(
+ FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("rating").build())
+ .addFeatures(
+ FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("null_value").build())
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("driver_id", intValue(1)))
+ .setProject("driver_project")
+ .build();
+ }
+
+ private static GetOnlineFeaturesResponse getFakeResponse() {
+ return GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("driver_id", intValue(1))
+ .putStatuses("driver_id", FieldStatus.PRESENT)
+ .putFields("driver:name", strValue("david"))
+ .putStatuses("driver:name", FieldStatus.PRESENT)
+ .putFields("driver:rating", intValue(3))
+ .putStatuses("driver:rating", FieldStatus.PRESENT)
+ .putFields("driver:null_value", Value.newBuilder().build())
+ .putStatuses("driver:null_value", FieldStatus.NULL_VALUE)
+ .build())
+ .build();
+ }
+
+ private static Value strValue(String val) {
+ return Value.newBuilder().setStringVal(val).build();
+ }
+
+ private static Value intValue(int val) {
+ return Value.newBuilder().setInt32Val(val).build();
+ }
+}
diff --git a/java/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java b/java/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java
new file mode 100644
index 00000000000..1592e20664d
--- /dev/null
+++ b/java/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java
@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gojek.feast;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.TextFormat;
+import feast.common.models.FeatureV2;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.NullSource;
+
+class RequestUtilTest {
+
+ private static Stream provideValidFeatureRefs() {
+ return Stream.of(
+ Arguments.of(
+ Arrays.asList("driver:driver_id"),
+ Arrays.asList(
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("driver")
+ .setName("driver_id")
+ .build())));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideValidFeatureRefs")
+ void createFeatureRefs_ShouldReturnFeaturesForValidFeatureRefs(
+ List input, List expected) {
+ List actual = RequestUtil.createFeatureRefs(input);
+ // Order of the actual and expected FeatureTables do no not matter
+ actual.sort(Comparator.comparing(FeatureReferenceV2::getName));
+ expected.sort(Comparator.comparing(FeatureReferenceV2::getName));
+ assertEquals(expected.size(), actual.size());
+ for (int i = 0; i < expected.size(); i++) {
+ String expectedString = TextFormat.printer().printToString(expected.get(i));
+ String actualString = TextFormat.printer().printToString(actual.get(i));
+ assertEquals(expectedString, actualString);
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideValidFeatureRefs")
+ void renderFeatureRef_ShouldReturnFeatureRefString(
+ List expected, List input) {
+ input = input.stream().map(ref -> ref.toBuilder().build()).collect(Collectors.toList());
+ List actual =
+ input.stream().map(ref -> FeatureV2.getFeatureStringRef(ref)).collect(Collectors.toList());
+ assertEquals(expected.size(), actual.size());
+ for (int i = 0; i < expected.size(); i++) {
+ assertEquals(expected.get(i), actual.get(i));
+ }
+ }
+
+ private static Stream provideInvalidFeatureRefs() {
+ return Stream.of(Arguments.of(ImmutableList.of("project/feature")));
+ }
+
+ private static Stream provideMissingFeatureTableFeatureRefs() {
+ return Stream.of(Arguments.of(ImmutableList.of("feature")));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideInvalidFeatureRefs")
+ void createFeatureRefs_ShouldThrowExceptionForProjectInFeatureRefs(List input) {
+ assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideMissingFeatureTableFeatureRefs")
+ void createFeatureRefs_ShouldThrowExceptionForMissingFeatureTableInFeatureRefs(
+ List input) {
+ assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input));
+ }
+
+ @ParameterizedTest
+ @NullSource
+ void createFeatureRefs_ShouldThrowExceptionForNullFeatureRefs(List input) {
+ assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input));
+ }
+}
diff --git a/java/serving/.dockerignore b/java/serving/.dockerignore
new file mode 100644
index 00000000000..441c3d1a50c
--- /dev/null
+++ b/java/serving/.dockerignore
@@ -0,0 +1,3 @@
+*/.gradle/
+.idea/
+*/build/
\ No newline at end of file
diff --git a/java/serving/.gitignore b/java/serving/.gitignore
new file mode 100644
index 00000000000..6c6b6d8d8f8
--- /dev/null
+++ b/java/serving/.gitignore
@@ -0,0 +1,37 @@
+### Scratch files ###
+scratch*
+
+### Local Environment ###
+*local*.env
+
+### Gradle ###
+.gradle
+**/build/
+!gradle/wrapper/gradle-wrapper.jar
+feast-serving.jar
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+/out/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+## Feast Temporary Files ##
+/temp/
\ No newline at end of file
diff --git a/java/serving/README.md b/java/serving/README.md
new file mode 100644
index 00000000000..cce8c7d6e29
--- /dev/null
+++ b/java/serving/README.md
@@ -0,0 +1,101 @@
+### Getting Started Guide for Feast Serving Developers
+
+Pre-requisites:
+
+- [Maven](https://maven.apache.org/install.html) build tool version 3.6.x
+- A running Feast Core instance
+- A running Store instance e.g. local Redis Store instance
+
+From the Feast project root directory, run the following Maven command to start Feast Serving gRPC service running on port 6566 locally:
+
+```bash
+# Assumptions:
+# - Local Feast Core is running on localhost:6565
+# Uses configuration from serving/src/main/resources/application.yml
+mvn -pl serving spring-boot:run -Dspring-boot.run.arguments=\
+--feast.core-host=localhost,\
+--feast.core-port=6565
+```
+
+If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) installed, you can check that Feast Serving is running
+```
+grpc_cli ls localhost:6566
+grpc_cli call localhost:6566 GetFeastServingVersion ''
+grpc_cli call localhost:6566 GetFeastServingType ''
+```
+
+```bash
+grpc_cli call localhost:6565 ApplyFeatureSet '
+feature_set {
+ name: "driver"
+ entities {
+ name: "driver_id"
+ value_type: STRING
+ }
+ features {
+ name: "city"
+ value_type: STRING
+ }
+ features {
+ name: "booking_completed_count"
+ value_type: INT64
+ }
+ source {
+ type: KAFKA
+ kafka_source_config {
+ bootstrap_servers: "localhost:9092"
+ }
+ }
+}
+'
+
+grpc_cli call localhost:6565 GetFeatureSets '
+filter {
+ feature_set_name: "driver"
+}
+'
+
+grpc_cli call localhost:6566 GetBatchFeatures '
+feature_sets {
+ name: "driver"
+ feature_names: "booking_completed_count"
+ max_age {
+ seconds: 86400
+ }
+}
+entity_dataset {
+ entity_names: "driver_id"
+ entity_dataset_rows {
+ entity_timestamp {
+ seconds: 1569873954
+ }
+ }
+}
+'
+```
+
+```
+python3 <
+
+
+ 4.0.0
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+
+
+ feast-serving
+ Feast Serving
+ Feature serving API service
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+ 11
+
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+ false
+
+
+
+ build-info
+
+ build-info
+
+
+
+
+
+
+
+
+
+
+ dev.feast
+ datatypes-java
+ ${project.version}
+
+
+
+
+ dev.feast
+ feast-storage-api
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-redis
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-bigtable
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-cassandra
+ ${project.version}
+
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ org.springframework.boot
+ spring-boot-configuration-processor
+ true
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+
+ org.springframework.boot
+ spring-boot-starter-log4j2
+
+
+ org.apache.logging.log4j
+ log4j-web
+
+
+
+ org.springframework.boot
+ spring-boot-devtools
+ true
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter-actuator
+
+
+
+
+ org.springframework.boot
+ spring-boot-test
+ 2.3.1.RELEASE
+ test
+
+
+
+
+ io.grpc
+ grpc-services
+
+
+
+ io.grpc
+ grpc-stub
+
+
+
+ com.google.protobuf
+ protobuf-java-util
+
+
+
+
+ com.google.guava
+ guava
+
+
+
+ joda-time
+ joda-time
+
+
+
+ io.jaegertracing
+ jaeger-client
+ 1.3.2
+
+
+ io.opentracing
+ opentracing-api
+ 0.33.0
+
+
+ io.opentracing
+ opentracing-noop
+ 0.33.0
+
+
+ io.opentracing.contrib
+ opentracing-grpc
+ 0.2.3
+
+
+
+
+ io.prometheus
+ simpleclient
+ 0.8.0
+
+
+
+
+ io.prometheus
+ simpleclient_hotspot
+ 0.8.0
+
+
+
+
+ io.prometheus
+ simpleclient_servlet
+ 0.8.0
+
+
+ io.prometheus
+ simpleclient_spring_boot
+ 0.8.0
+
+
+
+ com.google.auto.value
+ auto-value-annotations
+ 1.6.6
+
+
+
+
+
+ io.grpc
+ grpc-testing
+
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+ org.junit.vintage
+ junit-vintage-engine
+
+
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
+
+
+ org.apache.avro
+ avro
+ 1.10.2
+
+
+
+
+ org.apache.arrow
+ arrow-java-root
+ 5.0.0
+ pom
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+ 5.0.0
+
+
+
+
+ org.apache.arrow
+ arrow-memory
+ 5.0.0
+ pom
+
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+ 5.0.0
+ runtime
+
+
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+
+
+
+ com.github.kstyrc
+ embedded-redis
+ test
+
+
+ jakarta.validation
+ jakarta.validation-api
+ ${jakarta.validation.api.version}
+
+
+ net.devh
+ grpc-server-spring-boot-starter
+ ${grpc.spring.boot.starter.version}
+
+
+ org.testcontainers
+ testcontainers
+ 1.16.2
+ test
+
+
+ org.testcontainers
+ junit-jupiter
+ 1.16.2
+ test
+
+
+ org.awaitility
+ awaitility
+ 3.0.0
+ test
+
+
+ com.squareup.okhttp
+ okhttp
+ 2.7.4
+ test
+
+
+
+
+
+ profile-local
+
+
+ !ci
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-resources-plugin
+ 3.1.0
+
+
+ @
+
+ false
+
+
+
+
+
+
+
+ src/main/resources
+ true
+
+
+
+
+
+
diff --git a/java/serving/src/main/java/feast/serving/ServingApplication.java b/java/serving/src/main/java/feast/serving/ServingApplication.java
new file mode 100644
index 00000000000..ab036d04d18
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/ServingApplication.java
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving;
+
+import feast.serving.config.FeastProperties;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
+import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+
+@SpringBootApplication(
+ exclude = {
+ DataSourceAutoConfiguration.class,
+ DataSourceTransactionManagerAutoConfiguration.class,
+ HibernateJpaAutoConfiguration.class
+ })
+@EnableConfigurationProperties(FeastProperties.class)
+public class ServingApplication {
+ public static void main(String[] args) {
+ SpringApplication.run(ServingApplication.class, args);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/config/FeastProperties.java b/java/serving/src/main/java/feast/serving/config/FeastProperties.java
new file mode 100644
index 00000000000..493f8f9ed62
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/config/FeastProperties.java
@@ -0,0 +1,365 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.config;
+
+// Feast configuration properties that maps Feast configuration from default application.yml file to
+// a Java object.
+// https://www.baeldung.com/configuration-properties-in-spring-boot
+// https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties
+
+import feast.common.logging.config.LoggingProperties;
+import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
+import feast.storage.connectors.redis.retriever.RedisStoreConfig;
+import io.lettuce.core.ReadFrom;
+import java.time.Duration;
+import java.util.*;
+import javax.annotation.PostConstruct;
+import javax.validation.*;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.info.BuildProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+
+/** Feast Serving properties. */
+@ComponentScan("feast.common.logging")
+@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true)
+public class FeastProperties {
+
+ /**
+ * Instantiates a new Feast Serving properties.
+ *
+ * @param buildProperties the build properties
+ */
+ @Autowired
+ public FeastProperties(BuildProperties buildProperties) {
+ setVersion(buildProperties.getVersion());
+ }
+
+ /** Instantiates a new Feast class. */
+ public FeastProperties() {}
+
+ /* Feast Serving build version */
+ @NotBlank private String version = "unknown";
+
+ @NotBlank private String registry;
+
+ public String getRegistry() {
+ return registry;
+ }
+
+ public void setRegistry(final String registry) {
+ this.registry = registry;
+ }
+
+ private String transformationServiceEndpoint;
+
+ public String getTransformationServiceEndpoint() {
+ return transformationServiceEndpoint;
+ }
+
+ public void setTransformationServiceEndpoint(final String transformationServiceEndpoint) {
+ this.transformationServiceEndpoint = transformationServiceEndpoint;
+ }
+
+ /**
+ * Finds and returns the active store
+ *
+ * @return Returns the {@link Store} model object
+ */
+ public Store getActiveStore() {
+ for (Store store : getStores()) {
+ if (activeStore.equals(store.getName())) {
+ return store;
+ }
+ }
+ throw new RuntimeException(
+ String.format("Active store is misconfigured. Could not find store: %s.", activeStore));
+ }
+
+ /**
+ * Set the name of the active store found in the "stores" configuration list
+ *
+ * @param activeStore String name to active store
+ */
+ public void setActiveStore(String activeStore) {
+ this.activeStore = activeStore;
+ }
+
+ /** Name of the active store configuration (only one store can be active at a time). */
+ @NotBlank private String activeStore;
+
+ /**
+ * Collection of store configurations. The active store is selected by the "activeStore" field.
+ */
+ private List stores = new ArrayList<>();
+
+ /* Metric tracing properties. */
+ private TracingProperties tracing;
+
+ /* Feast Audit Logging properties */
+ @NotNull private LoggingProperties logging;
+
+ @Bean
+ LoggingProperties loggingProperties() {
+ return getLogging();
+ }
+
+ /**
+ * Gets Serving store configuration as a list of {@link Store}.
+ *
+ * @return List of stores objects
+ */
+ public List getStores() {
+ return stores;
+ }
+
+ /**
+ * Gets Feast Serving build version.
+ *
+ * @return the build version
+ */
+ public String getVersion() {
+ return version;
+ }
+
+ /**
+ * Sets build version
+ *
+ * @param version the build version
+ */
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ /**
+ * Sets the collection of configured stores.
+ *
+ * @param stores List of {@link Store}
+ */
+ public void setStores(List stores) {
+ this.stores = stores;
+ }
+
+ /** Store configuration class for database that this Feast Serving uses. */
+ public static class Store {
+
+ private String name;
+
+ private String type;
+
+ private Map config = new HashMap<>();
+
+ /**
+ * Gets name of this store. This is unique to this specific instance.
+ *
+ * @return the name of the store
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets the name of this store.
+ *
+ * @param name the name of the store
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Gets the store type. Example are REDIS, REDIS_CLUSTER, BIGTABLE or CASSANDRA
+ *
+ * @return the store type as a String.
+ */
+ public StoreType getType() {
+ return StoreType.valueOf(this.type);
+ }
+
+ /**
+ * Sets the store type
+ *
+ * @param type the type
+ */
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ * Gets the configuration to this specific store. This is a map of strings. These options are
+ * unique to the store. Please see protos/feast/core/Store.proto for the store specific
+ * configuration options
+ *
+ * @return Returns the store specific configuration
+ */
+ public Map getConfig() {
+ return config;
+ }
+
+ public RedisClusterStoreConfig getRedisClusterConfig() {
+ return new RedisClusterStoreConfig(
+ this.config.get("connection_string"),
+ ReadFrom.valueOf(this.config.get("read_from")),
+ Duration.parse(this.config.get("timeout")));
+ }
+
+ public RedisStoreConfig getRedisConfig() {
+ return new RedisStoreConfig(
+ this.config.get("host"),
+ Integer.valueOf(this.config.get("port")),
+ Boolean.valueOf(this.config.getOrDefault("ssl", "false")),
+ this.config.getOrDefault("password", ""));
+ }
+
+ /**
+ * Sets the store config. Please protos/feast/core/Store.proto for the specific options for each
+ * store.
+ *
+ * @param config the config map
+ */
+ public void setConfig(Map config) {
+ this.config = config;
+ }
+ }
+
+ public enum StoreType {
+ REDIS,
+ REDIS_CLUSTER;
+ }
+
+ /**
+ * Gets tracing properties
+ *
+ * @return tracing properties
+ */
+ public TracingProperties getTracing() {
+ return tracing;
+ }
+
+ /**
+ * Sets the tracing configuration.
+ *
+ * @param tracing the tracing
+ */
+ public void setTracing(TracingProperties tracing) {
+ this.tracing = tracing;
+ }
+
+ /**
+ * Gets logging properties
+ *
+ * @return logging properties
+ */
+ public LoggingProperties getLogging() {
+ return logging;
+ }
+
+ /**
+ * Sets logging properties
+ *
+ * @param logging the logging properties
+ */
+ public void setLogging(LoggingProperties logging) {
+ this.logging = logging;
+ }
+
+ /** Trace metric collection properties */
+ public static class TracingProperties {
+
+ /** Tracing enabled/disabled */
+ private boolean enabled;
+
+ /** Name of tracer to use (only "jaeger") */
+ private String tracerName;
+
+ /** Service name uniquely identifies this Feast Serving deployment */
+ private String serviceName;
+
+ /**
+ * Is tracing enabled
+ *
+ * @return boolean flag
+ */
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ /**
+ * Sets tracing enabled or disabled.
+ *
+ * @param enabled flag
+ */
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ /**
+ * Gets tracer name ('jaeger')
+ *
+ * @return the tracer name
+ */
+ public String getTracerName() {
+ return tracerName;
+ }
+
+ /**
+ * Sets tracer name.
+ *
+ * @param tracerName the tracer name
+ */
+ public void setTracerName(String tracerName) {
+ this.tracerName = tracerName;
+ }
+
+ /**
+ * Gets the service name. The service name uniquely identifies this Feast serving instance.
+ *
+ * @return the service name
+ */
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ /**
+ * Sets service name.
+ *
+ * @param serviceName the service name
+ */
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+ }
+
+ /**
+ * Validates all FeastProperties. This method runs after properties have been initialized and
+ * individually and conditionally validates each class.
+ */
+ @PostConstruct
+ public void validate() {
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Validator validator = factory.getValidator();
+
+ // Validate root fields in FeastProperties
+ Set> violations = validator.validate(this);
+ if (!violations.isEmpty()) {
+ throw new ConstraintViolationException(violations);
+ }
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java
new file mode 100644
index 00000000000..295b263f664
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.config;
+
+import io.opentracing.Tracer;
+import io.opentracing.contrib.grpc.TracingServerInterceptor;
+import io.opentracing.noop.NoopTracerFactory;
+import io.prometheus.client.exporter.MetricsServlet;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.web.servlet.ServletRegistrationBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class InstrumentationConfig {
+
+ private FeastProperties feastProperties;
+
+ @Autowired
+ public InstrumentationConfig(FeastProperties feastProperties) {
+ this.feastProperties = feastProperties;
+ }
+
+ @Bean
+ public ServletRegistrationBean servletRegistrationBean() {
+ DefaultExports.initialize();
+ return new ServletRegistrationBean(new MetricsServlet(), "/metrics");
+ }
+
+ @Bean
+ public Tracer tracer() {
+ if (!feastProperties.getTracing().isEnabled()) {
+ return NoopTracerFactory.create();
+ }
+
+ if (!feastProperties.getTracing().getTracerName().equalsIgnoreCase("jaeger")) {
+ throw new IllegalArgumentException("Only 'jaeger' tracer is supported for now.");
+ }
+
+ return io.jaegertracing.Configuration.fromEnv(feastProperties.getTracing().getServiceName())
+ .getTracer();
+ }
+
+ @Bean
+ public TracingServerInterceptor tracingInterceptor(Tracer tracer) {
+ return TracingServerInterceptor.newBuilder().withTracer(tracer).build();
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java b/java/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java
new file mode 100644
index 00000000000..ce4fe134373
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.config;
+
+import java.util.List;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.http.converter.protobuf.ProtobufJsonFormatHttpMessageConverter;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+@Configuration
+public class ServingApiConfiguration implements WebMvcConfigurer {
+ @Autowired private ProtobufJsonFormatHttpMessageConverter protobufConverter;
+
+ @Bean
+ ProtobufJsonFormatHttpMessageConverter protobufHttpMessageConverter() {
+ return new ProtobufJsonFormatHttpMessageConverter();
+ }
+
+ @Override
+ public void configureMessageConverters(List> converters) {
+ converters.add(protobufConverter);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java
new file mode 100644
index 00000000000..3a7936cb578
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.config;
+
+import feast.serving.registry.LocalRegistryRepo;
+import feast.serving.service.OnlineServingServiceV2;
+import feast.serving.service.OnlineTransformationService;
+import feast.serving.service.ServingServiceV2;
+import feast.serving.specs.FeatureSpecRetriever;
+import feast.serving.specs.RegistryFeatureSpecRetriever;
+import feast.storage.api.retriever.OnlineRetrieverV2;
+import feast.storage.connectors.redis.retriever.*;
+import io.opentracing.Tracer;
+import java.nio.file.Paths;
+import org.slf4j.Logger;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ServingServiceConfigV2 {
+ private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class);
+
+ @Bean
+ public ServingServiceV2 registryBasedServingServiceV2(
+ FeastProperties feastProperties, Tracer tracer) {
+ final ServingServiceV2 servingService;
+ final FeastProperties.Store store = feastProperties.getActiveStore();
+
+ OnlineRetrieverV2 retrieverV2;
+ // TODO: Support more store types, and potentially use a plugin model here.
+ switch (store.getType()) {
+ case REDIS_CLUSTER:
+ RedisClientAdapter redisClusterClient =
+ RedisClusterClient.create(store.getRedisClusterConfig());
+ retrieverV2 = new OnlineRetriever(redisClusterClient, new EntityKeySerializerV2());
+ break;
+ case REDIS:
+ RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig());
+ log.info("Created EntityKeySerializerV2");
+ retrieverV2 = new OnlineRetriever(redisClient, new EntityKeySerializerV2());
+ break;
+ default:
+ throw new RuntimeException(
+ String.format(
+ "Unable to identify online store type: %s for Regsitry Backed Serving Service",
+ store.getType()));
+ }
+
+ final FeatureSpecRetriever featureSpecRetriever;
+ log.info("Created RegistryFeatureSpecRetriever");
+ log.info("Working Directory = " + System.getProperty("user.dir"));
+ final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry()));
+ featureSpecRetriever = new RegistryFeatureSpecRetriever(repo);
+
+ final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint();
+ final OnlineTransformationService onlineTransformationService =
+ new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
+
+ servingService =
+ new OnlineServingServiceV2(
+ retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
+
+ return servingService;
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/config/WebSecurityConfig.java b/java/serving/src/main/java/feast/serving/config/WebSecurityConfig.java
new file mode 100644
index 00000000000..04d3f4b5afb
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/config/WebSecurityConfig.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+/**
+ * WebSecurityConfig disables auto configuration of Spring HTTP Security and allows security methods
+ * to be overridden
+ */
+@Configuration
+@EnableWebSecurity
+public class WebSecurityConfig extends WebSecurityConfigurerAdapter {
+
+ /**
+ * Allows for custom web security rules to be applied.
+ *
+ * @param http {@link HttpSecurity} for configuring web based security
+ * @throws Exception exception
+ */
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+
+ // Bypasses security/authentication for the following paths
+ http.authorizeRequests()
+ .antMatchers("/actuator/**", "/metrics/**")
+ .permitAll()
+ .anyRequest()
+ .authenticated()
+ .and()
+ .csrf()
+ .disable();
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/java/serving/src/main/java/feast/serving/controller/HealthServiceController.java
new file mode 100644
index 00000000000..ef675d4c157
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/controller/HealthServiceController.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.controller;
+
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.serving.interceptors.GrpcMonitoringInterceptor;
+import feast.serving.service.ServingServiceV2;
+import io.grpc.health.v1.HealthGrpc.HealthImplBase;
+import io.grpc.health.v1.HealthProto.HealthCheckRequest;
+import io.grpc.health.v1.HealthProto.HealthCheckResponse;
+import io.grpc.health.v1.HealthProto.ServingStatus;
+import io.grpc.stub.StreamObserver;
+import net.devh.boot.grpc.server.service.GrpcService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+// Reference: https://github.com/grpc/grpc/blob/master/doc/health-checking.md
+
+@GrpcService(interceptors = {GrpcMonitoringInterceptor.class})
+public class HealthServiceController extends HealthImplBase {
+ private final ServingServiceV2 servingService;
+
+ @Autowired
+ public HealthServiceController(final ServingServiceV2 servingService) {
+ this.servingService = servingService;
+ }
+
+ @Override
+ public void check(
+ HealthCheckRequest request, StreamObserver responseObserver) {
+ // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g.
+ // if it's online service check that it the service can retrieve dummy/random
+ // feature table.
+ // Implement similarly for batch service.
+
+ try {
+ servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance());
+ responseObserver.onNext(
+ HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build());
+ } catch (Exception e) {
+ responseObserver.onNext(
+ HealthCheckResponse.newBuilder().setStatus(ServingStatus.NOT_SERVING).build());
+ }
+ responseObserver.onCompleted();
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java
new file mode 100644
index 00000000000..048a53fad1b
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java
@@ -0,0 +1,107 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.controller;
+
+import feast.common.logging.interceptors.GrpcMessageInterceptor;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase;
+import feast.serving.config.FeastProperties;
+import feast.serving.exception.SpecRetrievalException;
+import feast.serving.interceptors.GrpcMonitoringContext;
+import feast.serving.interceptors.GrpcMonitoringInterceptor;
+import feast.serving.service.ServingServiceV2;
+import feast.serving.util.RequestHelper;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import io.opentracing.Span;
+import io.opentracing.Tracer;
+import io.opentracing.contrib.grpc.TracingServerInterceptor;
+import net.devh.boot.grpc.server.service.GrpcService;
+import org.slf4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+
+@GrpcService(
+ interceptors = {
+ TracingServerInterceptor.class,
+ GrpcMessageInterceptor.class,
+ GrpcMonitoringInterceptor.class
+ })
+public class ServingServiceGRpcController extends ServingServiceImplBase {
+
+ private static final Logger log =
+ org.slf4j.LoggerFactory.getLogger(ServingServiceGRpcController.class);
+ private final ServingServiceV2 servingServiceV2;
+ private final String version;
+ private final Tracer tracer;
+
+ @Autowired
+ public ServingServiceGRpcController(
+ ServingServiceV2 servingServiceV2, FeastProperties feastProperties, Tracer tracer) {
+ this.servingServiceV2 = servingServiceV2;
+ this.version = feastProperties.getVersion();
+ this.tracer = tracer;
+ }
+
+ @Override
+ public void getFeastServingInfo(
+ GetFeastServingInfoRequest request,
+ StreamObserver responseObserver) {
+ GetFeastServingInfoResponse feastServingInfo = servingServiceV2.getFeastServingInfo(request);
+ feastServingInfo = feastServingInfo.toBuilder().setVersion(version).build();
+ responseObserver.onNext(feastServingInfo);
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void getOnlineFeaturesV2(
+ ServingAPIProto.GetOnlineFeaturesRequestV2 request,
+ StreamObserver responseObserver) {
+ try {
+ // authorize for the project in request object.
+ if (request.getProject() != null && !request.getProject().isEmpty()) {
+ // update monitoring context
+ GrpcMonitoringContext.getInstance().setProject(request.getProject());
+ }
+ RequestHelper.validateOnlineRequest(request);
+ Span span = tracer.buildSpan("getOnlineFeaturesV2").start();
+ GetOnlineFeaturesResponse onlineFeatures = servingServiceV2.getOnlineFeatures(request);
+ if (span != null) {
+ span.finish();
+ }
+ responseObserver.onNext(onlineFeatures);
+ responseObserver.onCompleted();
+ } catch (SpecRetrievalException e) {
+ log.error("Failed to retrieve specs in SpecService", e);
+ responseObserver.onError(
+ Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException());
+ } catch (AccessDeniedException e) {
+ log.info(String.format("User prevented from accessing one of the projects in request"));
+ responseObserver.onError(
+ Status.PERMISSION_DENIED
+ .withDescription(e.getMessage())
+ .withCause(e)
+ .asRuntimeException());
+ } catch (Exception e) {
+ log.warn("Failed to get Online Features", e);
+ responseObserver.onError(e);
+ }
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java b/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java
new file mode 100644
index 00000000000..8a198a82019
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.controller;
+
+import static feast.serving.util.mappers.ResponseJSONMapper.mapGetOnlineFeaturesResponse;
+
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.serving.config.FeastProperties;
+import feast.serving.service.ServingServiceV2;
+import feast.serving.util.RequestHelper;
+import java.util.List;
+import java.util.Map;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class ServingServiceRestController {
+
+ private final ServingServiceV2 servingService;
+ private final String version;
+
+ @Autowired
+ public ServingServiceRestController(
+ ServingServiceV2 servingService, FeastProperties feastProperties) {
+ this.servingService = servingService;
+ this.version = feastProperties.getVersion();
+ }
+
+ @RequestMapping(value = "/api/v1/info", produces = "application/json")
+ public GetFeastServingInfoResponse getInfo() {
+ GetFeastServingInfoResponse feastServingInfo =
+ servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance());
+ return feastServingInfo.toBuilder().setVersion(version).build();
+ }
+
+ @RequestMapping(
+ value = "/api/v1/features/online",
+ produces = "application/json",
+ consumes = "application/json")
+ public List> getOnlineFeatures(
+ @RequestBody GetOnlineFeaturesRequestV2 request) {
+ RequestHelper.validateOnlineRequest(request);
+ GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request);
+ return mapGetOnlineFeaturesResponse(onlineFeatures);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java b/java/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java
new file mode 100644
index 00000000000..fbcba969fa4
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.exception;
+
+/** Application-specific exception for any failure of retrieving feature/entity/storage spec. */
+public class SpecRetrievalException extends RuntimeException {
+ public SpecRetrievalException() {
+ super();
+ }
+
+ public SpecRetrievalException(String message) {
+ super(message);
+ }
+
+ public SpecRetrievalException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java b/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java
new file mode 100644
index 00000000000..48d8d76a914
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.interceptors;
+
+import java.util.Optional;
+
+public class GrpcMonitoringContext {
+ private static GrpcMonitoringContext INSTANCE;
+
+ final ThreadLocal project = new ThreadLocal();
+
+ private GrpcMonitoringContext() {}
+
+ public static GrpcMonitoringContext getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE = new GrpcMonitoringContext();
+ }
+
+ return INSTANCE;
+ }
+
+ public void setProject(String name) {
+ this.project.set(name);
+ }
+
+ public Optional getProject() {
+ return Optional.ofNullable(this.project.get());
+ }
+
+ public void clearProject() {
+ this.project.set(null);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java
new file mode 100644
index 00000000000..735f8c556d6
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.interceptors;
+
+import feast.serving.util.Metrics;
+import io.grpc.ForwardingServerCall.SimpleForwardingServerCall;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCall.Listener;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.Status;
+import java.util.Optional;
+
+/**
+ * GrpcMonitoringInterceptor intercepts GRPC calls to provide request latency histogram metrics in
+ * the Prometheus client.
+ */
+public class GrpcMonitoringInterceptor implements ServerInterceptor {
+
+ @Override
+ public Listener interceptCall(
+ ServerCall call, Metadata headers, ServerCallHandler next) {
+
+ long startCallMillis = System.currentTimeMillis();
+ String fullMethodName = call.getMethodDescriptor().getFullMethodName();
+ String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1);
+
+ GrpcMonitoringContext.getInstance().clearProject();
+
+ return next.startCall(
+ new SimpleForwardingServerCall(call) {
+ @Override
+ public void close(Status status, Metadata trailers) {
+ Optional projectName = GrpcMonitoringContext.getInstance().getProject();
+
+ Metrics.requestLatency
+ .labels(methodName, projectName.orElse(""))
+ .observe((System.currentTimeMillis() - startCallMillis) / 1000f);
+ Metrics.grpcRequestCount.labels(methodName, status.getCode().name()).inc();
+ super.close(status, trailers);
+ }
+ },
+ headers);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/java/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java
new file mode 100644
index 00000000000..4178541a0fe
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java
@@ -0,0 +1,121 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.registry;
+
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
+import feast.proto.core.RegistryProto;
+import feast.proto.serving.ServingAPIProto;
+import feast.serving.exception.SpecRetrievalException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class LocalRegistryRepo implements RegistryRepository {
+ private final RegistryProto.Registry registry;
+ private Map featureViewNameToSpec;
+ private Map
+ onDemandFeatureViewNameToSpec;
+
+ public LocalRegistryRepo(Path localRegistryPath) {
+ if (!localRegistryPath.toFile().exists()) {
+ throw new RuntimeException(
+ String.format("Local registry not found at path %s", localRegistryPath));
+ }
+ try {
+ final byte[] registryContents = Files.readAllBytes(localRegistryPath);
+ this.registry = RegistryProto.Registry.parseFrom(registryContents);
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ final RegistryProto.Registry registry = this.getRegistry();
+ List featureViewSpecs =
+ registry.getFeatureViewsList().stream()
+ .map(fv -> fv.getSpec())
+ .collect(Collectors.toList());
+ featureViewNameToSpec =
+ featureViewSpecs.stream()
+ .collect(
+ Collectors.toMap(FeatureViewProto.FeatureViewSpec::getName, Function.identity()));
+ List onDemandFeatureViewSpecs =
+ registry.getOnDemandFeatureViewsList().stream()
+ .map(odfv -> odfv.getSpec())
+ .collect(Collectors.toList());
+ onDemandFeatureViewNameToSpec =
+ onDemandFeatureViewSpecs.stream()
+ .collect(
+ Collectors.toMap(
+ OnDemandFeatureViewProto.OnDemandFeatureViewSpec::getName,
+ Function.identity()));
+ }
+
+ @Override
+ public RegistryProto.Registry getRegistry() {
+ return this.registry;
+ }
+
+ @Override
+ public FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ String featureViewName = featureReference.getFeatureTable();
+ if (featureViewNameToSpec.containsKey(featureViewName)) {
+ return featureViewNameToSpec.get(featureViewName);
+ }
+ throw new SpecRetrievalException(
+ String.format("Unable to find feature view with name: %s", featureViewName));
+ }
+
+ @Override
+ public FeatureProto.FeatureSpecV2 getFeatureSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ final FeatureViewProto.FeatureViewSpec spec =
+ this.getFeatureViewSpec(projectName, featureReference);
+ for (final FeatureProto.FeatureSpecV2 featureSpec : spec.getFeaturesList()) {
+ if (featureSpec.getName().equals(featureReference.getName())) {
+ return featureSpec;
+ }
+ }
+
+ throw new SpecRetrievalException(
+ String.format(
+ "Unable to find feature with name: %s in feature view: %s",
+ featureReference.getName(), featureReference.getFeatureTable()));
+ }
+
+ @Override
+ public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ String onDemandFeatureViewName = featureReference.getFeatureTable();
+ if (onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName)) {
+ return onDemandFeatureViewNameToSpec.get(onDemandFeatureViewName);
+ }
+ throw new SpecRetrievalException(
+ String.format(
+ "Unable to find on demand feature view with name: %s", onDemandFeatureViewName));
+ }
+
+ @Override
+ public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) {
+ String onDemandFeatureViewName = featureReference.getFeatureTable();
+ return onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/java/serving/src/main/java/feast/serving/registry/RegistryRepository.java
new file mode 100644
index 00000000000..21a21838aad
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/registry/RegistryRepository.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.registry;
+
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
+import feast.proto.core.RegistryProto;
+import feast.proto.serving.ServingAPIProto;
+
+/**
+ * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry
+ * object. This approach is needed for a feature store created using feast 0.10+.
+ */
+public interface RegistryRepository {
+ RegistryProto.Registry getRegistry();
+
+ FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ FeatureProto.FeatureSpecV2 getFeatureSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference);
+}
diff --git a/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java
new file mode 100644
index 00000000000..2992c5022d2
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java
@@ -0,0 +1,428 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import static feast.common.models.FeatureTable.getFeatureTableStringRef;
+
+import com.google.protobuf.Duration;
+import feast.common.models.FeatureV2;
+import feast.proto.serving.ServingAPIProto.FeastServingType;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse;
+import feast.proto.serving.TransformationServiceAPIProto.ValueType;
+import feast.proto.types.ValueProto;
+import feast.serving.exception.SpecRetrievalException;
+import feast.serving.specs.FeatureSpecRetriever;
+import feast.serving.util.Metrics;
+import feast.storage.api.retriever.Feature;
+import feast.storage.api.retriever.OnlineRetrieverV2;
+import io.grpc.Status;
+import io.opentracing.Span;
+import io.opentracing.Tracer;
+import java.io.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+
+public class OnlineServingServiceV2 implements ServingServiceV2 {
+
+ private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class);
+ private final Tracer tracer;
+ private final OnlineRetrieverV2 retriever;
+ private final FeatureSpecRetriever featureSpecRetriever;
+ private final OnlineTransformationService onlineTransformationService;
+
+ public OnlineServingServiceV2(
+ OnlineRetrieverV2 retriever,
+ Tracer tracer,
+ FeatureSpecRetriever featureSpecRetriever,
+ OnlineTransformationService onlineTransformationService) {
+ this.retriever = retriever;
+ this.tracer = tracer;
+ this.featureSpecRetriever = featureSpecRetriever;
+ this.onlineTransformationService = onlineTransformationService;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public GetFeastServingInfoResponse getFeastServingInfo(
+ GetFeastServingInfoRequest getFeastServingInfoRequest) {
+ return GetFeastServingInfoResponse.newBuilder()
+ .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE)
+ .build();
+ }
+
+ @Override
+ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 request) {
+ // Autofill default project if project is not specified
+ String projectName = request.getProject();
+ if (projectName.isEmpty()) {
+ projectName = "default";
+ }
+
+ // Split all feature references into non-ODFV (e.g. batch and stream) references and ODFV.
+ List allFeatureReferences = request.getFeaturesList();
+ List featureReferences =
+ allFeatureReferences.stream()
+ .filter(r -> !this.featureSpecRetriever.isOnDemandFeatureReference(r))
+ .collect(Collectors.toList());
+ List onDemandFeatureReferences =
+ allFeatureReferences.stream()
+ .filter(r -> this.featureSpecRetriever.isOnDemandFeatureReference(r))
+ .collect(Collectors.toList());
+
+ // Get the set of request data feature names and feature inputs from the ODFV references.
+ Pair, List> pair =
+ this.onlineTransformationService.extractRequestDataFeatureNamesAndOnDemandFeatureInputs(
+ onDemandFeatureReferences, projectName);
+ Set requestDataFeatureNames = pair.getLeft();
+ List onDemandFeatureInputs = pair.getRight();
+
+ // Add on demand feature inputs to list of feature references to retrieve.
+ Set addedFeatureReferences = new HashSet();
+ for (FeatureReferenceV2 onDemandFeatureInput : onDemandFeatureInputs) {
+ if (!featureReferences.contains(onDemandFeatureInput)) {
+ featureReferences.add(onDemandFeatureInput);
+ addedFeatureReferences.add(onDemandFeatureInput);
+ }
+ }
+
+ // Separate entity rows into entity data and request feature data.
+ Pair, Map>>
+ entityRowsAndRequestDataFeatures =
+ this.onlineTransformationService.separateEntityRows(requestDataFeatureNames, request);
+ List entityRows =
+ entityRowsAndRequestDataFeatures.getLeft();
+ Map> requestDataFeatures =
+ entityRowsAndRequestDataFeatures.getRight();
+ // TODO: error checking on lengths of lists in entityRows and requestDataFeatures
+
+ // Extract values and statuses to be used later in constructing FieldValues for the response.
+ // The online features retrieved will augment these two data structures.
+ List> values =
+ entityRows.stream().map(r -> new HashMap<>(r.getFieldsMap())).collect(Collectors.toList());
+ List> statuses =
+ entityRows.stream()
+ .map(
+ r ->
+ r.getFieldsMap().entrySet().stream()
+ .map(entry -> Pair.of(entry.getKey(), getMetadata(entry.getValue(), false)))
+ .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)))
+ .collect(Collectors.toList());
+
+ String finalProjectName = projectName;
+ Map featureMaxAges =
+ featureReferences.stream()
+ .distinct()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ ref -> this.featureSpecRetriever.getMaxAge(finalProjectName, ref)));
+ List entityNames =
+ featureReferences.stream()
+ .map(ref -> this.featureSpecRetriever.getEntitiesList(finalProjectName, ref))
+ .findFirst()
+ .get();
+
+ Map featureValueTypes =
+ featureReferences.stream()
+ .distinct()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ ref -> {
+ try {
+ return this.featureSpecRetriever
+ .getFeatureSpec(finalProjectName, ref)
+ .getValueType();
+ } catch (SpecRetrievalException e) {
+ return ValueProto.ValueType.Enum.INVALID;
+ }
+ }));
+
+ Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start();
+ if (storageRetrievalSpan != null) {
+ storageRetrievalSpan.setTag("entities", entityRows.size());
+ storageRetrievalSpan.setTag("features", featureReferences.size());
+ }
+ List> entityRowsFeatures =
+ retriever.getOnlineFeatures(projectName, entityRows, featureReferences, entityNames);
+ if (storageRetrievalSpan != null) {
+ storageRetrievalSpan.finish();
+ }
+
+ if (entityRowsFeatures.size() != entityRows.size()) {
+ throw Status.INTERNAL
+ .withDescription(
+ "The no. of FeatureRow obtained from OnlineRetriever"
+ + "does not match no. of entityRow passed.")
+ .asRuntimeException();
+ }
+
+ Span postProcessingSpan = tracer.buildSpan("postProcessing").start();
+
+ for (int i = 0; i < entityRows.size(); i++) {
+ GetOnlineFeaturesRequestV2.EntityRow entityRow = entityRows.get(i);
+ List curEntityRowFeatures = entityRowsFeatures.get(i);
+
+ Map featureReferenceFeatureMap =
+ getFeatureRefFeatureMap(curEntityRowFeatures);
+
+ Map rowValues = values.get(i);
+ Map rowStatuses = statuses.get(i);
+
+ for (FeatureReferenceV2 featureReference : featureReferences) {
+ if (featureReferenceFeatureMap.containsKey(featureReference)) {
+ Feature feature = featureReferenceFeatureMap.get(featureReference);
+
+ ValueProto.Value value =
+ feature.getFeatureValue(featureValueTypes.get(feature.getFeatureReference()));
+
+ Boolean isOutsideMaxAge =
+ checkOutsideMaxAge(
+ feature, entityRow, featureMaxAges.get(feature.getFeatureReference()));
+
+ if (value != null) {
+ rowValues.put(FeatureV2.getFeatureStringRef(feature.getFeatureReference()), value);
+ } else {
+ rowValues.put(
+ FeatureV2.getFeatureStringRef(feature.getFeatureReference()),
+ ValueProto.Value.newBuilder().build());
+ }
+
+ rowStatuses.put(
+ FeatureV2.getFeatureStringRef(feature.getFeatureReference()),
+ getMetadata(value, isOutsideMaxAge));
+ } else {
+ rowValues.put(
+ FeatureV2.getFeatureStringRef(featureReference),
+ ValueProto.Value.newBuilder().build());
+
+ rowStatuses.put(
+ FeatureV2.getFeatureStringRef(featureReference), getMetadata(null, false));
+ }
+ }
+ // Populate metrics/log request
+ populateCountMetrics(rowStatuses, projectName);
+ }
+
+ if (postProcessingSpan != null) {
+ postProcessingSpan.finish();
+ }
+
+ populateHistogramMetrics(entityRows, featureReferences, projectName);
+ populateFeatureCountMetrics(featureReferences, projectName);
+
+ // Handle ODFVs. For each ODFV reference, we send a TransformFeaturesRequest to the FTS.
+ // The request should contain the entity data, the retrieved features, and the request data.
+ if (!onDemandFeatureReferences.isEmpty()) {
+ // Augment values, which contains the entity data and retrieved features, with the request
+ // data. Also augment statuses.
+ for (int i = 0; i < values.size(); i++) {
+ Map rowValues = values.get(i);
+ Map rowStatuses = statuses.get(i);
+
+ for (Map.Entry> entry : requestDataFeatures.entrySet()) {
+ String key = entry.getKey();
+ List fieldValues = entry.getValue();
+ rowValues.put(key, fieldValues.get(i));
+ rowStatuses.put(key, GetOnlineFeaturesResponse.FieldStatus.PRESENT);
+ }
+ }
+
+ // Serialize the augmented values.
+ ValueType transformationInput =
+ this.onlineTransformationService.serializeValuesIntoArrowIPC(values);
+
+ // Send out requests to the FTS and process the responses.
+ Set onDemandFeatureStringReferences =
+ onDemandFeatureReferences.stream()
+ .map(r -> FeatureV2.getFeatureStringRef(r))
+ .collect(Collectors.toSet());
+ for (FeatureReferenceV2 featureReference : onDemandFeatureReferences) {
+ String onDemandFeatureViewName = featureReference.getFeatureTable();
+ TransformFeaturesRequest transformFeaturesRequest =
+ TransformFeaturesRequest.newBuilder()
+ .setOnDemandFeatureViewName(onDemandFeatureViewName)
+ .setProject(projectName)
+ .setTransformationInput(transformationInput)
+ .build();
+
+ TransformFeaturesResponse transformFeaturesResponse =
+ this.onlineTransformationService.transformFeatures(transformFeaturesRequest);
+
+ this.onlineTransformationService.processTransformFeaturesResponse(
+ transformFeaturesResponse,
+ onDemandFeatureViewName,
+ onDemandFeatureStringReferences,
+ values,
+ statuses);
+ }
+
+ // Remove all features that were added as inputs for ODFVs.
+ Set addedFeatureStringReferences =
+ addedFeatureReferences.stream()
+ .map(r -> FeatureV2.getFeatureStringRef(r))
+ .collect(Collectors.toSet());
+ for (int i = 0; i < values.size(); i++) {
+ Map rowValues = values.get(i);
+ Map rowStatuses = statuses.get(i);
+ List keysToRemove =
+ rowValues.keySet().stream()
+ .filter(k -> addedFeatureStringReferences.contains(k))
+ .collect(Collectors.toList());
+ for (String key : keysToRemove) {
+ rowValues.remove(key);
+ rowStatuses.remove(key);
+ }
+ }
+ }
+
+ // Build response field values from entityValuesMap and entityStatusesMap
+ // Response field values should be in the same order as the entityRows provided by the user.
+ List fieldValuesList =
+ IntStream.range(0, entityRows.size())
+ .mapToObj(
+ entityRowIdx ->
+ GetOnlineFeaturesResponse.FieldValues.newBuilder()
+ .putAllFields(values.get(entityRowIdx))
+ .putAllStatuses(statuses.get(entityRowIdx))
+ .build())
+ .collect(Collectors.toList());
+
+ return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build();
+ }
+
+ private static Map getFeatureRefFeatureMap(List features) {
+ return features.stream()
+ .collect(Collectors.toMap(Feature::getFeatureReference, Function.identity()));
+ }
+
+ /**
+ * Generate Field level Status metadata for the given valueMap.
+ *
+ * @param value value to generate metadata for.
+ * @param isOutsideMaxAge whether the given valueMap contains values with age outside
+ * FeatureTable's max age.
+ * @return a 1:1 map keyed by field name containing field status metadata instead of values in the
+ * given valueMap.
+ */
+ private static GetOnlineFeaturesResponse.FieldStatus getMetadata(
+ ValueProto.Value value, boolean isOutsideMaxAge) {
+
+ if (value == null) {
+ return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND;
+ } else if (isOutsideMaxAge) {
+ return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE;
+ } else if (value.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) {
+ return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE;
+ }
+ return GetOnlineFeaturesResponse.FieldStatus.PRESENT;
+ }
+
+ /**
+ * Determine if the feature data in the given feature row is outside maxAge. Data is outside
+ * maxAge to be when the difference ingestion time set in feature row and the retrieval time set
+ * in entity row exceeds FeatureTable max age.
+ *
+ * @param feature contains the ingestion timing and feature data.
+ * @param entityRow contains the retrieval timing of when features are pulled.
+ * @param maxAge feature's max age.
+ */
+ private static boolean checkOutsideMaxAge(
+ Feature feature, GetOnlineFeaturesRequestV2.EntityRow entityRow, Duration maxAge) {
+
+ if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set
+ return false;
+ }
+
+ long givenTimestamp = entityRow.getTimestamp().getSeconds();
+ if (givenTimestamp == 0) {
+ givenTimestamp = System.currentTimeMillis() / 1000;
+ }
+ long timeDifference = givenTimestamp - feature.getEventTimestamp().getSeconds();
+ return timeDifference > maxAge.getSeconds();
+ }
+
+ /**
+ * Populate histogram metrics that can be used for analysing online retrieval calls
+ *
+ * @param entityRows entity rows provided in request
+ * @param featureReferences feature references provided in request
+ * @param project project name provided in request
+ */
+ private void populateHistogramMetrics(
+ List entityRows,
+ List featureReferences,
+ String project) {
+ Metrics.requestEntityCountDistribution
+ .labels(project)
+ .observe(Double.valueOf(entityRows.size()));
+ Metrics.requestFeatureCountDistribution
+ .labels(project)
+ .observe(Double.valueOf(featureReferences.size()));
+
+ long countDistinctFeatureTables =
+ featureReferences.stream()
+ .map(featureReference -> getFeatureTableStringRef(project, featureReference))
+ .distinct()
+ .count();
+ Metrics.requestFeatureTableCountDistribution
+ .labels(project)
+ .observe(Double.valueOf(countDistinctFeatureTables));
+ }
+
+ /**
+ * Populate count metrics that can be used for analysing online retrieval calls
+ *
+ * @param statusMap Statuses of features which have been requested
+ * @param project Project where request for features was called from
+ */
+ private void populateCountMetrics(
+ Map statusMap, String project) {
+ statusMap.forEach(
+ (featureRefString, status) -> {
+ if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) {
+ Metrics.notFoundKeyCount.labels(project, featureRefString).inc();
+ }
+ if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) {
+ Metrics.staleKeyCount.labels(project, featureRefString).inc();
+ }
+ });
+ }
+
+ private void populateFeatureCountMetrics(
+ List featureReferences, String project) {
+ featureReferences.forEach(
+ featureReference ->
+ Metrics.requestFeatureCount
+ .labels(project, FeatureV2.getFeatureStringRef(featureReference))
+ .inc());
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java b/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java
new file mode 100644
index 00000000000..541fe46898a
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java
@@ -0,0 +1,412 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import com.google.protobuf.ByteString;
+import feast.common.models.FeatureV2;
+import feast.proto.core.DataSourceProto;
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse;
+import feast.proto.serving.TransformationServiceAPIProto.ValueType;
+import feast.proto.serving.TransformationServiceGrpc;
+import feast.proto.types.ValueProto;
+import feast.serving.specs.FeatureSpecRetriever;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Status;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.*;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.tomcat.util.http.fileupload.ByteArrayOutputStream;
+import org.slf4j.Logger;
+
+public class OnlineTransformationService implements TransformationService {
+
+ private static final Logger log =
+ org.slf4j.LoggerFactory.getLogger(OnlineTransformationService.class);
+ private final TransformationServiceGrpc.TransformationServiceBlockingStub stub;
+ private final FeatureSpecRetriever featureSpecRetriever;
+ static final int INT64_BITWIDTH = 64;
+ static final int INT32_BITWIDTH = 32;
+
+ public OnlineTransformationService(
+ String transformationServiceEndpoint, FeatureSpecRetriever featureSpecRetriever) {
+ if (transformationServiceEndpoint != null) {
+ final ManagedChannel channel =
+ ManagedChannelBuilder.forTarget(transformationServiceEndpoint).usePlaintext().build();
+ this.stub = TransformationServiceGrpc.newBlockingStub(channel);
+ } else {
+ this.stub = null;
+ }
+ this.featureSpecRetriever = featureSpecRetriever;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public TransformFeaturesResponse transformFeatures(
+ TransformFeaturesRequest transformFeaturesRequest) {
+ return this.stub.transformFeatures(transformFeaturesRequest);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Pair, List>
+ extractRequestDataFeatureNamesAndOnDemandFeatureInputs(
+ List onDemandFeatureReferences, String projectName) {
+ Set requestDataFeatureNames = new HashSet();
+ List onDemandFeatureInputs =
+ new ArrayList();
+ for (ServingAPIProto.FeatureReferenceV2 featureReference : onDemandFeatureReferences) {
+ OnDemandFeatureViewProto.OnDemandFeatureViewSpec onDemandFeatureViewSpec =
+ this.featureSpecRetriever.getOnDemandFeatureViewSpec(projectName, featureReference);
+ Map inputs =
+ onDemandFeatureViewSpec.getInputsMap();
+
+ for (OnDemandFeatureViewProto.OnDemandInput input : inputs.values()) {
+ OnDemandFeatureViewProto.OnDemandInput.InputCase inputCase = input.getInputCase();
+ switch (inputCase) {
+ case REQUEST_DATA_SOURCE:
+ DataSourceProto.DataSource requestDataSource = input.getRequestDataSource();
+ DataSourceProto.DataSource.RequestDataOptions requestDataOptions =
+ requestDataSource.getRequestDataOptions();
+ Set requestDataNames = requestDataOptions.getSchemaMap().keySet();
+ requestDataFeatureNames.addAll(requestDataNames);
+ break;
+ case FEATURE_VIEW:
+ FeatureViewProto.FeatureView featureView = input.getFeatureView();
+ FeatureViewProto.FeatureViewSpec featureViewSpec = featureView.getSpec();
+ String featureViewName = featureViewSpec.getName();
+ for (FeatureProto.FeatureSpecV2 featureSpec : featureViewSpec.getFeaturesList()) {
+ String featureName = featureSpec.getName();
+ ServingAPIProto.FeatureReferenceV2 onDemandFeatureInput =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable(featureViewName)
+ .setName(featureName)
+ .build();
+ onDemandFeatureInputs.add(onDemandFeatureInput);
+ }
+ break;
+ default:
+ throw Status.INTERNAL
+ .withDescription(
+ "OnDemandInput proto input field has an unexpected type: " + inputCase)
+ .asRuntimeException();
+ }
+ }
+ }
+ Pair, List> pair =
+ new ImmutablePair, List>(
+ requestDataFeatureNames, onDemandFeatureInputs);
+ return pair;
+ }
+
+ /** {@inheritDoc} */
+ public Pair<
+ List,
+ Map>>
+ separateEntityRows(
+ Set requestDataFeatureNames, ServingAPIProto.GetOnlineFeaturesRequestV2 request) {
+ // Separate entity rows into entity data and request feature data.
+ List entityRows =
+ new ArrayList();
+ Map> requestDataFeatures =
+ new HashMap>();
+
+ for (ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow :
+ request.getEntityRowsList()) {
+ Map fieldsMap = new HashMap();
+
+ for (Map.Entry entry : entityRow.getFieldsMap().entrySet()) {
+ String key = entry.getKey();
+ ValueProto.Value value = entry.getValue();
+
+ if (requestDataFeatureNames.contains(key)) {
+ if (!requestDataFeatures.containsKey(key)) {
+ requestDataFeatures.put(key, new ArrayList());
+ }
+ requestDataFeatures.get(key).add(value);
+ } else {
+ fieldsMap.put(key, value);
+ }
+ }
+
+ // Construct new entity row containing the extracted entity data, if necessary.
+ if (!fieldsMap.isEmpty()) {
+ ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow newEntityRow =
+ ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+ .setTimestamp(entityRow.getTimestamp())
+ .putAllFields(fieldsMap)
+ .build();
+ entityRows.add(newEntityRow);
+ }
+ }
+
+ Pair<
+ List,
+ Map>>
+ pair =
+ new ImmutablePair<
+ List,
+ Map>>(entityRows, requestDataFeatures);
+ return pair;
+ }
+
+ /** {@inheritDoc} */
+ public void processTransformFeaturesResponse(
+ feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse
+ transformFeaturesResponse,
+ String onDemandFeatureViewName,
+ Set onDemandFeatureStringReferences,
+ List> values,
+ List> statuses) {
+ try {
+ BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+ ArrowFileReader reader =
+ new ArrowFileReader(
+ new ByteArrayReadableSeekableByteChannel(
+ transformFeaturesResponse
+ .getTransformationOutput()
+ .getArrowValue()
+ .toByteArray()),
+ allocator);
+ reader.loadNextBatch();
+ VectorSchemaRoot readBatch = reader.getVectorSchemaRoot();
+ Schema responseSchema = readBatch.getSchema();
+ List responseFields = responseSchema.getFields();
+
+ for (Field field : responseFields) {
+ String columnName = field.getName();
+ String fullFeatureName = onDemandFeatureViewName + ":" + columnName;
+ ArrowType columnType = field.getType();
+
+ // The response will contain all features for the specified ODFV, so we
+ // skip the features that were not requested.
+ if (!onDemandFeatureStringReferences.contains(fullFeatureName)) {
+ continue;
+ }
+
+ FieldVector fieldVector = readBatch.getVector(field);
+ int valueCount = fieldVector.getValueCount();
+
+ // TODO: support all Feast types
+ // TODO: clean up the switch statement
+ if (columnType instanceof ArrowType.Int) {
+ int bitWidth = ((ArrowType.Int) columnType).getBitWidth();
+ switch (bitWidth) {
+ case INT64_BITWIDTH:
+ for (int i = 0; i < valueCount; i++) {
+ long int64Value = ((BigIntVector) fieldVector).get(i);
+ Map rowValues = values.get(i);
+ Map rowStatuses =
+ statuses.get(i);
+ ValueProto.Value value =
+ ValueProto.Value.newBuilder().setInt64Val(int64Value).build();
+ rowValues.put(fullFeatureName, value);
+ rowStatuses.put(
+ fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT);
+ }
+ break;
+ case INT32_BITWIDTH:
+ for (int i = 0; i < valueCount; i++) {
+ int intValue = ((IntVector) fieldVector).get(i);
+ Map rowValues = values.get(i);
+ Map rowStatuses =
+ statuses.get(i);
+ ValueProto.Value value =
+ ValueProto.Value.newBuilder().setInt32Val(intValue).build();
+ rowValues.put(fullFeatureName, value);
+ rowStatuses.put(
+ fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT);
+ }
+ break;
+ default:
+ throw Status.INTERNAL
+ .withDescription(
+ "Column "
+ + columnName
+ + " is of type ArrowType.Int but has bitWidth "
+ + bitWidth
+ + " which cannot be handled.")
+ .asRuntimeException();
+ }
+ } else if (columnType instanceof ArrowType.FloatingPoint) {
+ FloatingPointPrecision precision = ((ArrowType.FloatingPoint) columnType).getPrecision();
+ switch (precision) {
+ case DOUBLE:
+ for (int i = 0; i < valueCount; i++) {
+ double doubleValue = ((Float8Vector) fieldVector).get(i);
+ Map rowValues = values.get(i);
+ Map rowStatuses =
+ statuses.get(i);
+ ValueProto.Value value =
+ ValueProto.Value.newBuilder().setDoubleVal(doubleValue).build();
+ rowValues.put(fullFeatureName, value);
+ rowStatuses.put(
+ fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT);
+ }
+ break;
+ case SINGLE:
+ for (int i = 0; i < valueCount; i++) {
+ float floatValue = ((Float4Vector) fieldVector).get(i);
+ Map rowValues = values.get(i);
+ Map rowStatuses =
+ statuses.get(i);
+ ValueProto.Value value =
+ ValueProto.Value.newBuilder().setFloatVal(floatValue).build();
+ rowValues.put(fullFeatureName, value);
+ rowStatuses.put(
+ fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT);
+ }
+ break;
+ default:
+ throw Status.INTERNAL
+ .withDescription(
+ "Column "
+ + columnName
+ + " is of type ArrowType.FloatingPoint but has precision "
+ + precision
+ + " which cannot be handled.")
+ .asRuntimeException();
+ }
+ }
+ }
+ } catch (IOException e) {
+ log.info(e.toString());
+ throw Status.INTERNAL
+ .withDescription(
+ "Unable to correctly process transform features response: " + e.toString())
+ .asRuntimeException();
+ }
+ }
+
+ /** {@inheritDoc} */
+ public ValueType serializeValuesIntoArrowIPC(List> values) {
+ // In order to be serialized correctly, the data must be packaged in a VectorSchemaRoot.
+ // We first construct all the columns.
+ Map columnNameToColumn = new HashMap();
+ BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+ Map firstAugmentedRowValues = values.get(0);
+ for (Map.Entry entry : firstAugmentedRowValues.entrySet()) {
+ // The Python FTS does not expect full feature names, so we extract the feature name.
+ String columnName = FeatureV2.getFeatureName(entry.getKey());
+ ValueProto.Value.ValCase valCase = entry.getValue().getValCase();
+ FieldVector column;
+ // TODO: support all Feast types
+ switch (valCase) {
+ case INT32_VAL:
+ column = new IntVector(columnName, allocator);
+ break;
+ case INT64_VAL:
+ column = new BigIntVector(columnName, allocator);
+ break;
+ case DOUBLE_VAL:
+ column = new Float8Vector(columnName, allocator);
+ break;
+ case FLOAT_VAL:
+ column = new Float4Vector(columnName, allocator);
+ break;
+ default:
+ throw Status.INTERNAL
+ .withDescription(
+ "Column " + columnName + " has a type that is currently not handled: " + valCase)
+ .asRuntimeException();
+ }
+ column.allocateNew();
+ columnNameToColumn.put(columnName, column);
+ }
+
+ // Add the data, row by row.
+ for (int i = 0; i < values.size(); i++) {
+ Map augmentedRowValues = values.get(i);
+
+ for (Map.Entry entry : augmentedRowValues.entrySet()) {
+ String columnName = FeatureV2.getFeatureName(entry.getKey());
+ ValueProto.Value value = entry.getValue();
+ ValueProto.Value.ValCase valCase = value.getValCase();
+ FieldVector column = columnNameToColumn.get(columnName);
+ // TODO: support all Feast types
+ switch (valCase) {
+ case INT32_VAL:
+ ((IntVector) column).setSafe(i, value.getInt32Val());
+ break;
+ case INT64_VAL:
+ ((BigIntVector) column).setSafe(i, value.getInt64Val());
+ break;
+ case DOUBLE_VAL:
+ ((Float8Vector) column).setSafe(i, value.getDoubleVal());
+ break;
+ case FLOAT_VAL:
+ ((Float4Vector) column).setSafe(i, value.getFloatVal());
+ break;
+ default:
+ throw Status.INTERNAL
+ .withDescription(
+ "Column "
+ + columnName
+ + " has a type that is currently not handled: "
+ + valCase)
+ .asRuntimeException();
+ }
+ }
+ }
+
+ // Construct the VectorSchemaRoot.
+ List columnFields = new ArrayList();
+ List columns = new ArrayList();
+ for (FieldVector column : columnNameToColumn.values()) {
+ column.setValueCount(values.size());
+ columnFields.add(column.getField());
+ columns.add(column);
+ }
+ VectorSchemaRoot schemaRoot = new VectorSchemaRoot(columnFields, columns);
+
+ // Serialize the VectorSchemaRoot into Arrow IPC format.
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ArrowFileWriter writer = new ArrowFileWriter(schemaRoot, null, Channels.newChannel(out));
+ try {
+ writer.start();
+ writer.writeBatch();
+ writer.end();
+ } catch (IOException e) {
+ log.info(e.toString());
+ throw Status.INTERNAL
+ .withDescription(
+ "ArrowFileWriter could not write properly; failed with error: " + e.toString())
+ .asRuntimeException();
+ }
+ byte[] byteData = out.toByteArray();
+ ByteString inputData = ByteString.copyFrom(byteData);
+ ValueType transformationInput = ValueType.newBuilder().setArrowValue(inputData).build();
+ return transformationInput;
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java b/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java
new file mode 100644
index 00000000000..05acb31b78e
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+
+public interface ServingServiceV2 {
+ /**
+ * Get information about the Feast serving deployment.
+ *
+ * For Bigquery deployments, this includes the default job staging location to load
+ * intermediate files to. Otherwise, this method only returns the current Feast Serving backing
+ * store type.
+ *
+ * @param getFeastServingInfoRequest {@link ServingAPIProto.GetFeastServingInfoRequest}
+ * @return {@link ServingAPIProto.GetFeastServingInfoResponse}
+ */
+ ServingAPIProto.GetFeastServingInfoResponse getFeastServingInfo(
+ ServingAPIProto.GetFeastServingInfoRequest getFeastServingInfoRequest);
+
+ /**
+ * Get features from an online serving store, given a list of {@link
+ * feast.proto.serving.ServingAPIProto.FeatureReferenceV2}s to retrieve, and list of {@link
+ * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s to join the
+ * retrieved values to.
+ *
+ *
Features can be queried across feature tables, but each {@link
+ * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} must contain all
+ * entities for all feature tables included in the request.
+ *
+ *
This request is fulfilled synchronously.
+ *
+ * @param getFeaturesRequest {@link GetOnlineFeaturesRequestV2} containing list of {@link
+ * feast.proto.serving.ServingAPIProto.FeatureReferenceV2}s to retrieve and list of {@link
+ * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s to join the
+ * retrieved values to.
+ * @return {@link GetOnlineFeaturesResponse} with list of {@link
+ * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues} for each {@link
+ * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} supplied.
+ */
+ GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 getFeaturesRequest);
+}
diff --git a/java/serving/src/main/java/feast/serving/service/TransformationService.java b/java/serving/src/main/java/feast/serving/service/TransformationService.java
new file mode 100644
index 00000000000..caa52793020
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/service/TransformationService.java
@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest;
+import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse;
+import feast.proto.serving.TransformationServiceAPIProto.ValueType;
+import feast.proto.types.ValueProto;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
+
+public interface TransformationService {
+ /**
+ * Apply on demand transformations for the specified ODFVs.
+ *
+ * @param transformFeaturesRequest proto containing the ODFV references and necessary data
+ * @return a proto object containing the response
+ */
+ TransformFeaturesResponse transformFeatures(TransformFeaturesRequest transformFeaturesRequest);
+
+ /**
+ * Extract the set of request data feature names and the list of on demand feature inputs from a
+ * list of ODFV references.
+ *
+ * @param onDemandFeatureReferences list of ODFV references to be parsed
+ * @param projectName project name
+ * @return a pair containing the set of request data feature names and list of on demand feature
+ * inputs
+ */
+ Pair, List>
+ extractRequestDataFeatureNamesAndOnDemandFeatureInputs(
+ List onDemandFeatureReferences, String projectName);
+
+ /**
+ * Separate the entity rows of a request into entity data and request feature data.
+ *
+ * @param requestDataFeatureNames set of feature names for the request data
+ * @param request the GetOnlineFeaturesRequestV2 containing the entity rows
+ * @return a pair containing the set of request data feature names and list of on demand feature
+ * inputs
+ */
+ Pair, Map>>
+ separateEntityRows(Set requestDataFeatureNames, GetOnlineFeaturesRequestV2 request);
+
+ /**
+ * Process a response from the feature transformation server by augmenting the given lists of
+ * field maps and status maps with the correct fields from the response.
+ *
+ * @param transformFeaturesResponse response to be processed
+ * @param onDemandFeatureViewName name of ODFV to which the response corresponds
+ * @param onDemandFeatureStringReferences set of all ODFV references that should be kept
+ * @param values list of field maps to be augmented with additional fields from the response
+ * @param statuses list of status maps to be augmented
+ */
+ void processTransformFeaturesResponse(
+ TransformFeaturesResponse transformFeaturesResponse,
+ String onDemandFeatureViewName,
+ Set onDemandFeatureStringReferences,
+ List> values,
+ List> statuses);
+
+ /**
+ * Serialize data into Arrow IPC format, to be sent to the Python feature transformation server.
+ *
+ * @param values list of field maps to be serialized
+ * @return the data packaged into a ValueType proto object
+ */
+ ValueType serializeValuesIntoArrowIPC(List> values);
+}
diff --git a/java/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java b/java/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java
new file mode 100644
index 00000000000..57e931ccf9d
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.specs;
+
+import com.google.protobuf.Duration;
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
+import feast.proto.serving.ServingAPIProto;
+import java.util.List;
+
+public interface FeatureSpecRetriever {
+
+ Duration getMaxAge(String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ List getEntitiesList(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ FeatureProto.FeatureSpecV2 getFeatureSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+ boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference);
+}
diff --git a/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java b/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java
new file mode 100644
index 00000000000..540e7f504e5
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.specs;
+
+import com.google.protobuf.Duration;
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
+import feast.proto.serving.ServingAPIProto;
+import feast.serving.registry.RegistryRepository;
+import java.util.List;
+
+public class RegistryFeatureSpecRetriever implements FeatureSpecRetriever {
+ private final RegistryRepository registryRepository;
+
+ public RegistryFeatureSpecRetriever(RegistryRepository registryRepository) {
+ this.registryRepository = registryRepository;
+ }
+
+ @Override
+ public Duration getMaxAge(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository.getFeatureViewSpec(projectName, featureReference).getTtl();
+ }
+
+ @Override
+ public List getEntitiesList(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository
+ .getFeatureViewSpec(projectName, featureReference)
+ .getEntitiesList();
+ }
+
+ @Override
+ public FeatureProto.FeatureSpecV2 getFeatureSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository.getFeatureSpec(projectName, featureReference);
+ }
+
+ @Override
+ public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository.getFeatureViewSpec(projectName, featureReference);
+ }
+
+ @Override
+ public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+ String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository.getOnDemandFeatureViewSpec(projectName, featureReference);
+ }
+
+ @Override
+ public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) {
+ return this.registryRepository.isOnDemandFeatureReference(featureReference);
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/util/Metrics.java b/java/serving/src/main/java/feast/serving/util/Metrics.java
new file mode 100644
index 00000000000..dca2b5e82aa
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/util/Metrics.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util;
+
+import io.prometheus.client.Counter;
+import io.prometheus.client.Histogram;
+
+public class Metrics {
+
+ public static final Histogram requestLatency =
+ Histogram.build()
+ .name("request_latency_seconds")
+ .subsystem("feast_serving")
+ .help("Request latency in seconds")
+ .labelNames("method", "project")
+ .register();
+
+ public static final Histogram requestEntityCountDistribution =
+ Histogram.build()
+ .buckets(1, 2, 5, 10, 20, 50, 100, 200)
+ .name("request_entity_count_distribution")
+ .subsystem("feast_serving")
+ .help("Number of entity rows per request")
+ .labelNames("project")
+ .register();
+
+ public static final Histogram requestFeatureCountDistribution =
+ Histogram.build()
+ .buckets(1, 2, 5, 10, 15, 20, 30, 50)
+ .name("request_feature_count_distribution")
+ .subsystem("feast_serving")
+ .help("Number of feature rows per request")
+ .labelNames("project")
+ .register();
+
+ public static final Histogram requestFeatureTableCountDistribution =
+ Histogram.build()
+ .buckets(1, 2, 5, 10, 20)
+ .name("request_feature_table_count_distribution")
+ .subsystem("feast_serving")
+ .help("Number of feature tables per request")
+ .labelNames("project")
+ .register();
+
+ public static final Counter requestFeatureCount =
+ Counter.build()
+ .name("request_feature_count")
+ .subsystem("feast_serving")
+ .help("number of feature rows requested")
+ .labelNames("project", "feature_name")
+ .register();
+
+ public static final Counter notFoundKeyCount =
+ Counter.build()
+ .name("not_found_feature_count")
+ .subsystem("feast_serving")
+ .help("number requested feature rows that were not found")
+ .labelNames("project", "feature_name")
+ .register();
+
+ public static final Counter staleKeyCount =
+ Counter.build()
+ .name("stale_feature_count")
+ .subsystem("feast_serving")
+ .help("number requested feature rows that were stale")
+ .labelNames("project", "feature_name")
+ .register();
+
+ public static final Counter grpcRequestCount =
+ Counter.build()
+ .name("grpc_request_count")
+ .subsystem("feast_serving")
+ .help("number of grpc requests served")
+ .labelNames("method", "status_code")
+ .register();
+}
diff --git a/java/serving/src/main/java/feast/serving/util/RequestHelper.java b/java/serving/src/main/java/feast/serving/util/RequestHelper.java
new file mode 100644
index 00000000000..4d478f430f2
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/util/RequestHelper.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+
+public class RequestHelper {
+
+ public static void validateOnlineRequest(GetOnlineFeaturesRequestV2 request) {
+ // All EntityRows should not be empty
+ if (request.getEntityRowsCount() <= 0) {
+ throw new IllegalArgumentException("Entity value must be provided");
+ }
+ // All FeatureReferences should have FeatureTable name and Feature name
+ for (FeatureReferenceV2 featureReference : request.getFeaturesList()) {
+ validateOnlineRequestFeatureReference(featureReference);
+ }
+ }
+
+ public static void validateOnlineRequestFeatureReference(FeatureReferenceV2 featureReference) {
+ if (featureReference.getFeatureTable().isEmpty()) {
+ throw new IllegalArgumentException("FeatureTable name must be provided in FeatureReference");
+ }
+ if (featureReference.getName().isEmpty()) {
+ throw new IllegalArgumentException("Feature name must be provided in FeatureReference");
+ }
+ }
+}
diff --git a/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java
new file mode 100644
index 00000000000..238df549513
--- /dev/null
+++ b/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util.mappers;
+
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.proto.types.ValueProto.Value;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+// ResponseJSONMapper maps GRPC Response types to more human readable JSON responses
+public class ResponseJSONMapper {
+
+ public static List> mapGetOnlineFeaturesResponse(
+ GetOnlineFeaturesResponse response) {
+ return response.getFieldValuesList().stream()
+ .map(fieldValues -> convertFieldValuesToMap(fieldValues))
+ .collect(Collectors.toList());
+ }
+
+ private static Map convertFieldValuesToMap(FieldValues fieldValues) {
+ return fieldValues.getFieldsMap().entrySet().stream()
+ .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue())));
+ }
+
+ private static Object extractValue(Value value) {
+ switch (value.getValCase().getNumber()) {
+ case 1:
+ return value.getBytesVal();
+ case 2:
+ return value.getStringVal();
+ case 3:
+ return value.getInt32Val();
+ case 4:
+ return value.getInt64Val();
+ case 5:
+ return value.getDoubleVal();
+ case 6:
+ return value.getFloatVal();
+ case 7:
+ return value.getBoolVal();
+ case 11:
+ return value.getBytesListVal();
+ case 12:
+ return value.getStringListVal();
+ case 13:
+ return value.getInt32ListVal();
+ case 14:
+ return value.getInt64ListVal();
+ case 15:
+ return value.getDoubleListVal();
+ case 16:
+ return value.getFloatListVal();
+ case 17:
+ return value.getBoolListVal();
+ default:
+ return null;
+ }
+ }
+}
diff --git a/java/serving/src/main/resources/application.yml b/java/serving/src/main/resources/application.yml
new file mode 100644
index 00000000000..087bb46f2fe
--- /dev/null
+++ b/java/serving/src/main/resources/application.yml
@@ -0,0 +1,64 @@
+feast:
+ # Indicates the active store. Only a single store in the last can be active at one time. In the future this key
+ # will be deprecated in order to allow multiple stores to be served from a single serving instance
+ active_store: online
+
+ # List of store configurations
+ stores:
+ # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options
+ - name: online # Name of the store (referenced by active_store)
+ type: REDIS # Type of the store. REDIS, REDIS_CLUSTER are available options
+ config: # Store specific configuration. See
+ host: localhost
+ port: 6379
+ # Subscriptions indicate which feature sets needs to be retrieved and used to populate this store
+ - name: online_cluster
+ type: REDIS_CLUSTER
+ config: # Store specific configuration.
+ # Connection string specifies the host:port of Redis instances in the redis cluster.
+ connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005"
+ read_from: MASTER
+ # Redis operation timeout in ISO-8601 format
+ timeout: PT0.5S
+ tracing:
+ # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls
+ # which can be useful to debug performance issues and perform benchmarking
+ enabled: false
+ # Only Jaeger tracer is supported currently
+ # https://opentracing.io/docs/supported-tracers/
+ tracer-name: jaeger
+ # The service name identifier for the tracing data
+ service-name: feast_serving
+
+ logging:
+ # Audit logging provides a machine readable structured JSON log that can give better
+ # insight into what is happening in Feast.
+ audit:
+ # Whether audit logging is enabled.
+ enabled: true
+ # Whether to enable message level (ie request/response) audit logging
+ messageLogging:
+ enabled: false
+ # Logging forwarder currently provides a machine readable structured JSON log to an
+ # external fluentd service that can give better insight into what is happening in Feast.
+ # Accepts console / fluentd as destination
+ destination: console
+ fluentdHost: localhost
+ fluentdPort: 24224
+
+grpc:
+ server:
+ # The port number Feast Serving GRPC service should listen on
+ # It is set default to 6566 so it does not conflict with the GRPC server on Feast Core
+ # which defaults to port 6565
+ port: ${GRPC_PORT:6566}
+ security:
+ enabled: false
+ certificateChain: server.crt
+ privateKey: server.key
+
+server:
+ # The port number on which the Tomcat webserver that serves REST API endpoints should listen
+ # It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core
+ # if both Feast Core and Serving are running on the same machine
+ port: ${SERVER_PORT:8081}
diff --git a/java/serving/src/main/resources/banner.txt b/java/serving/src/main/resources/banner.txt
new file mode 100644
index 00000000000..44aa9c4cad9
--- /dev/null
+++ b/java/serving/src/main/resources/banner.txt
@@ -0,0 +1,14 @@
+
+███████╗███████╗ █████╗ ███████╗████████╗
+██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝
+█████╗ █████╗ ███████║███████╗ ██║
+██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║
+██║ ███████╗██║ ██║███████║ ██║
+╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝
+
+███████╗███████╗██████╗ ██╗ ██╗██╗███╗ ██╗ ██████╗
+██╔════╝██╔════╝██╔══██╗██║ ██║██║████╗ ██║██╔════╝
+███████╗█████╗ ██████╔╝██║ ██║██║██╔██╗ ██║██║ ███╗
+╚════██║██╔══╝ ██╔══██╗╚██╗ ██╔╝██║██║╚██╗██║██║ ██║
+███████║███████╗██║ ██║ ╚████╔╝ ██║██║ ╚████║╚██████╔╝
+╚══════╝╚══════╝╚═╝ ╚═╝ ╚═══╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝
diff --git a/java/serving/src/main/resources/log4j2.xml b/java/serving/src/main/resources/log4j2.xml
new file mode 100644
index 00000000000..c75c2db13cc
--- /dev/null
+++ b/java/serving/src/main/resources/log4j2.xml
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex
+
+
+ {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java
new file mode 100644
index 00000000000..b7f02315213
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java
@@ -0,0 +1,96 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.controller;
+
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.types.ValueProto.Value;
+import feast.serving.config.FeastProperties;
+import feast.serving.service.ServingServiceV2;
+import io.grpc.stub.StreamObserver;
+import io.jaegertracing.Configuration;
+import io.opentracing.Tracer;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.springframework.security.core.Authentication;
+
+public class ServingServiceGRpcControllerTest {
+
+ @Mock private ServingServiceV2 mockServingServiceV2;
+
+ @Mock private StreamObserver mockStreamObserver;
+
+ private GetOnlineFeaturesRequestV2 validRequest;
+
+ private ServingServiceGRpcController service;
+
+ @Mock private Authentication authentication;
+
+ @Before
+ public void setUp() {
+ initMocks(this);
+
+ validRequest =
+ GetOnlineFeaturesRequestV2.newBuilder()
+ .addFeatures(
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature1")
+ .build())
+ .addFeatures(
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature2")
+ .build())
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", Value.newBuilder().setInt64Val(1).build())
+ .putFields("entity2", Value.newBuilder().setInt64Val(1).build()))
+ .build();
+ }
+
+ private ServingServiceGRpcController getServingServiceGRpcController(boolean enableAuth) {
+ Tracer tracer = Configuration.fromEnv("dummy").getTracer();
+ FeastProperties feastProperties = new FeastProperties();
+
+ return new ServingServiceGRpcController(mockServingServiceV2, feastProperties, tracer);
+ }
+
+ @Test
+ public void shouldPassValidRequestAsIs() {
+ service = getServingServiceGRpcController(false);
+ service.getOnlineFeaturesV2(validRequest, mockStreamObserver);
+ Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest);
+ }
+
+ @Test
+ public void shouldCallOnErrorIfEntityDatasetIsNotSet() {
+ service = getServingServiceGRpcController(false);
+ GetOnlineFeaturesRequestV2 missingEntityName =
+ GetOnlineFeaturesRequestV2.newBuilder(validRequest).clearEntityRows().build();
+ service.getOnlineFeaturesV2(missingEntityName, mockStreamObserver);
+ Mockito.verify(mockStreamObserver).onError(Mockito.any(IllegalArgumentException.class));
+ }
+}
diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java
new file mode 100644
index 00000000000..6fd065d2640
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java
@@ -0,0 +1,206 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.it;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.Timestamp;
+import com.squareup.okhttp.OkHttpClient;
+import com.squareup.okhttp.Request;
+import com.squareup.okhttp.Response;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingServiceGrpc;
+import feast.serving.util.DataGenerator;
+import io.grpc.ManagedChannel;
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.web.server.LocalServerPort;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+import org.testcontainers.containers.DockerComposeContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+@ActiveProfiles("it")
+@SpringBootTest(
+ webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
+ properties = {
+ "feast.registry:src/test/resources/docker-compose/feast10/registry.db",
+ })
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS)
+@Testcontainers
+public class ServingServiceRedisIT {
+
+ public static final Logger log = LoggerFactory.getLogger(ServingServiceRedisIT.class);
+
+ static ServingServiceGrpc.ServingServiceBlockingStub servingStub;
+
+ static final int FEAST_SERVING_PORT = 6568;
+
+ @LocalServerPort private int metricsPort;
+
+ @ClassRule @Container
+ public static DockerComposeContainer environment =
+ new DockerComposeContainer(
+ new File("src/test/resources/docker-compose/docker-compose-redis-it.yml"))
+ .withExposedService("redis", 6379)
+ .withOptions()
+ .waitingFor(
+ "materialize",
+ Wait.forLogMessage(".*Materialization finished.*\\n", 1)
+ .withStartupTimeout(Duration.ofMinutes(5)));
+
+ @DynamicPropertySource
+ static void initialize(DynamicPropertyRegistry registry) {
+ registry.add("grpc.server.port", () -> FEAST_SERVING_PORT);
+
+ registry.add("feast.stores[0].name", () -> "online");
+ registry.add("feast.stores[0].type", () -> "REDIS");
+ registry.add("feast.stores[0].config.host", () -> environment.getServiceHost("redis", 6379));
+ registry.add("feast.stores[0].config.port", () -> environment.getServicePort("redis", 6379));
+ }
+
+ @BeforeAll
+ static void globalSetup() {
+ servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null);
+ }
+
+ @AfterAll
+ static void tearDown() throws Exception {
+ ((ManagedChannel) servingStub.getChannel()).shutdown().awaitTermination(10, TimeUnit.SECONDS);
+ }
+
+ private GetOnlineFeaturesRequestV2 buildOnlineRequest(int driverId) {
+ // getOnlineFeatures Information
+ String projectName = "feast_project";
+ String entityName = "driver_id";
+
+ // Instantiate EntityRows
+ final Timestamp timestamp = Timestamp.getDefaultInstance();
+ GetOnlineFeaturesRequestV2.EntityRow entityRow1 =
+ DataGenerator.createEntityRow(
+ entityName, DataGenerator.createInt64Value(driverId), timestamp.getSeconds());
+ ImmutableList entityRows = ImmutableList.of(entityRow1);
+
+ // Instantiate FeatureReferences
+ ServingAPIProto.FeatureReferenceV2 feature1Reference =
+ DataGenerator.createFeatureReference("driver_hourly_stats", "conv_rate");
+ ServingAPIProto.FeatureReferenceV2 feature2Reference =
+ DataGenerator.createFeatureReference("driver_hourly_stats", "avg_daily_trips");
+ ImmutableList featureReferences =
+ ImmutableList.of(feature1Reference, feature2Reference);
+
+ // Build GetOnlineFeaturesRequestV2
+ return TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows);
+ }
+
+ @Test
+ @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
+ public void shouldGetOnlineFeatures() {
+ GetOnlineFeaturesResponse featureResponse =
+ servingStub.getOnlineFeaturesV2(buildOnlineRequest(1005));
+
+ assertEquals(1, featureResponse.getFieldValuesCount());
+
+ final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0);
+ for (final String key :
+ ImmutableList.of(
+ "driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate", "driver_id")) {
+ assertTrue(fieldValue.containsFields(key));
+ assertTrue(fieldValue.containsStatuses(key));
+ assertEquals(
+ GetOnlineFeaturesResponse.FieldStatus.PRESENT, fieldValue.getStatusesOrThrow(key));
+ }
+
+ assertEquals(
+ 500, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val());
+ assertEquals(1005, fieldValue.getFieldsOrThrow("driver_id").getInt64Val());
+ assertEquals(
+ 0.5, fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), 0.0001);
+ }
+
+ @Test
+ @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
+ public void shouldGetOnlineFeaturesWithOutsideMaxAgeStatus() {
+ GetOnlineFeaturesResponse featureResponse =
+ servingStub.getOnlineFeaturesV2(buildOnlineRequest(1001));
+
+ assertEquals(1, featureResponse.getFieldValuesCount());
+
+ final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0);
+ for (final String key :
+ ImmutableList.of("driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate")) {
+ assertTrue(fieldValue.containsFields(key));
+ assertTrue(fieldValue.containsStatuses(key));
+ assertEquals(
+ GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE,
+ fieldValue.getStatusesOrThrow(key));
+ }
+
+ assertEquals(
+ 100, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val());
+ assertEquals(1001, fieldValue.getFieldsOrThrow("driver_id").getInt64Val());
+ assertEquals(
+ 0.1, fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), 0.0001);
+ }
+
+ @Test
+ @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
+ public void shouldGetOnlineFeaturesWithNotFoundStatus() {
+ GetOnlineFeaturesResponse featureResponse =
+ servingStub.getOnlineFeaturesV2(buildOnlineRequest(-1));
+
+ assertEquals(1, featureResponse.getFieldValuesCount());
+
+ final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0);
+ for (final String key :
+ ImmutableList.of("driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate")) {
+ assertTrue(fieldValue.containsFields(key));
+ assertTrue(fieldValue.containsStatuses(key));
+ assertEquals(
+ GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, fieldValue.getStatusesOrThrow(key));
+ }
+ }
+
+ @Test
+ @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
+ public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException {
+ Request request =
+ new Request.Builder()
+ .url(String.format("http://localhost:%d/metrics", metricsPort))
+ .get()
+ .build();
+ Response response = new OkHttpClient().newCall(request).execute();
+ assertTrue(response.isSuccessful());
+ assertFalse(response.body().string().isEmpty());
+ }
+}
diff --git a/java/serving/src/test/java/feast/serving/it/TestUtils.java b/java/serving/src/test/java/feast/serving/it/TestUtils.java
new file mode 100644
index 00000000000..fb88b2fb372
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/it/TestUtils.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.it;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingServiceGrpc;
+import io.grpc.Channel;
+import io.grpc.ManagedChannelBuilder;
+import java.util.*;
+
+public class TestUtils {
+
+ public static ServingServiceGrpc.ServingServiceBlockingStub getServingServiceStub(
+ boolean isSecure, int feastServingPort, Map options) {
+ Channel secureChannel =
+ ManagedChannelBuilder.forAddress("localhost", feastServingPort).usePlaintext().build();
+ return ServingServiceGrpc.newBlockingStub(secureChannel);
+ }
+
+ public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest(
+ String projectName,
+ List featureReferences,
+ List entityRows) {
+ return GetOnlineFeaturesRequestV2.newBuilder()
+ .setProject(projectName)
+ .addAllFeatures(featureReferences)
+ .addAllEntityRows(entityRows)
+ .build();
+ }
+}
diff --git a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java
new file mode 100644
index 00000000000..d64cd07fb4c
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java
@@ -0,0 +1,395 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import static feast.serving.util.DataGenerator.*;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import com.google.protobuf.Duration;
+import com.google.protobuf.Timestamp;
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureViewProto;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.proto.types.ValueProto;
+import feast.serving.registry.LocalRegistryRepo;
+import feast.serving.specs.FeatureSpecRetriever;
+import feast.serving.specs.RegistryFeatureSpecRetriever;
+import feast.storage.api.retriever.Feature;
+import feast.storage.api.retriever.ProtoFeature;
+import feast.storage.connectors.redis.retriever.OnlineRetriever;
+import io.opentracing.Tracer;
+import io.opentracing.Tracer.SpanBuilder;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+public class OnlineServingServiceTest {
+
+ @Mock LocalRegistryRepo registryRepo;
+ @Mock Tracer tracer;
+ @Mock OnlineRetriever retrieverV2;
+ private String transformationServiceEndpoint;
+
+ private OnlineServingServiceV2 onlineServingServiceV2;
+
+ List mockedFeatureRows;
+ List featureSpecs;
+
+ @Before
+ public void setUp() {
+ initMocks(this);
+ FeatureSpecRetriever featureSpecRetriever = new RegistryFeatureSpecRetriever(registryRepo);
+ OnlineTransformationService onlineTransformationService =
+ new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
+ onlineServingServiceV2 =
+ new OnlineServingServiceV2(
+ retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
+
+ mockedFeatureRows = new ArrayList<>();
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build(),
+ Timestamp.newBuilder().setSeconds(100).build(),
+ createStrValue("1")));
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build(),
+ Timestamp.newBuilder().setSeconds(100).build(),
+ createStrValue("2")));
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build(),
+ Timestamp.newBuilder().setSeconds(100).build(),
+ createStrValue("3")));
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build(),
+ Timestamp.newBuilder().setSeconds(100).build(),
+ createStrValue("4")));
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_3")
+ .build(),
+ Timestamp.newBuilder().setSeconds(100).build(),
+ createStrValue("5")));
+ mockedFeatureRows.add(
+ new ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build(),
+ Timestamp.newBuilder().setSeconds(50).build(),
+ createStrValue("6")));
+
+ featureSpecs = new ArrayList<>();
+ featureSpecs.add(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_1")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build());
+ featureSpecs.add(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_2")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build());
+ }
+
+ @Test
+ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() {
+ String projectName = "default";
+ ServingAPIProto.FeatureReferenceV2 featureReference1 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build();
+ ServingAPIProto.FeatureReferenceV2 featureReference2 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build();
+ List featureReferences =
+ List.of(featureReference1, featureReference2);
+ GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences);
+
+ List entityKeyList1 = new ArrayList<>();
+ List entityKeyList2 = new ArrayList<>();
+ entityKeyList1.add(mockedFeatureRows.get(0));
+ entityKeyList1.add(mockedFeatureRows.get(1));
+ entityKeyList2.add(mockedFeatureRows.get(2));
+ entityKeyList2.add(mockedFeatureRows.get(3));
+
+ List> featureRows = List.of(entityKeyList1, entityKeyList2);
+
+ when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows);
+ when(registryRepo.getFeatureViewSpec(any(), any())).thenReturn(getFeatureViewSpec());
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference()))
+ .thenReturn(featureSpecs.get(0));
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference()))
+ .thenReturn(featureSpecs.get(1));
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference()))
+ .thenReturn(featureSpecs.get(0));
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference()))
+ .thenReturn(featureSpecs.get(1));
+
+ when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class));
+
+ GetOnlineFeaturesResponse expected =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(1))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("a"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createStrValue("1"))
+ .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_2", createStrValue("2"))
+ .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT)
+ .build())
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(2))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("b"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createStrValue("3"))
+ .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_2", createStrValue("4"))
+ .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT)
+ .build())
+ .build();
+ GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request);
+ assertThat(actual, equalTo(expected));
+ }
+
+ @Test
+ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() {
+ String projectName = "default";
+ ServingAPIProto.FeatureReferenceV2 featureReference1 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build();
+ ServingAPIProto.FeatureReferenceV2 featureReference2 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build();
+ List featureReferences =
+ List.of(featureReference1, featureReference2);
+ GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences);
+
+ List entityKeyList1 = new ArrayList<>();
+ List entityKeyList2 = new ArrayList<>();
+ entityKeyList1.add(mockedFeatureRows.get(0));
+ entityKeyList1.add(mockedFeatureRows.get(1));
+ entityKeyList2.add(mockedFeatureRows.get(4));
+
+ List> featureRows = List.of(entityKeyList1, entityKeyList2);
+
+ when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows);
+ when(registryRepo.getFeatureViewSpec(any(), any())).thenReturn(getFeatureViewSpec());
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference()))
+ .thenReturn(featureSpecs.get(0));
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference()))
+ .thenReturn(featureSpecs.get(1));
+
+ when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class));
+
+ GetOnlineFeaturesResponse expected =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(1))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("a"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createStrValue("1"))
+ .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_2", createStrValue("2"))
+ .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT)
+ .build())
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(2))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("b"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createEmptyValue())
+ .putStatuses("featuretable_1:feature_1", FieldStatus.NOT_FOUND)
+ .putFields("featuretable_1:feature_2", createEmptyValue())
+ .putStatuses("featuretable_1:feature_2", FieldStatus.NOT_FOUND)
+ .build())
+ .build();
+ GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request);
+ assertThat(actual, equalTo(expected));
+ }
+
+ @Test
+ public void shouldReturnResponseWithValuesAndMetadataIfMaxAgeIsExceeded() {
+ String projectName = "default";
+ ServingAPIProto.FeatureReferenceV2 featureReference1 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build();
+ ServingAPIProto.FeatureReferenceV2 featureReference2 =
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build();
+ List featureReferences =
+ List.of(featureReference1, featureReference2);
+ GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences);
+
+ List entityKeyList1 = new ArrayList<>();
+ List entityKeyList2 = new ArrayList<>();
+ entityKeyList1.add(mockedFeatureRows.get(5));
+ entityKeyList1.add(mockedFeatureRows.get(1));
+ entityKeyList2.add(mockedFeatureRows.get(5));
+ entityKeyList2.add(mockedFeatureRows.get(1));
+
+ List> featureRows = List.of(entityKeyList1, entityKeyList2);
+
+ when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows);
+ when(registryRepo.getFeatureViewSpec(any(), any()))
+ .thenReturn(
+ FeatureViewProto.FeatureViewSpec.newBuilder()
+ .setName("featuretable_1")
+ .addEntities("entity1")
+ .addEntities("entity2")
+ .addFeatures(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_1")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build())
+ .addFeatures(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_2")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build())
+ .setTtl(Duration.newBuilder().setSeconds(1))
+ .build());
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference()))
+ .thenReturn(featureSpecs.get(1));
+ when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference()))
+ .thenReturn(featureSpecs.get(0));
+
+ when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class));
+
+ GetOnlineFeaturesResponse expected =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(1))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("a"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createStrValue("6"))
+ .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE)
+ .putFields("featuretable_1:feature_2", createStrValue("2"))
+ .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT)
+ .build())
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", createInt64Value(2))
+ .putStatuses("entity1", FieldStatus.PRESENT)
+ .putFields("entity2", createStrValue("b"))
+ .putStatuses("entity2", FieldStatus.PRESENT)
+ .putFields("featuretable_1:feature_1", createStrValue("6"))
+ .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE)
+ .putFields("featuretable_1:feature_2", createStrValue("2"))
+ .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT)
+ .build())
+ .build();
+ GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request);
+ assertThat(actual, equalTo(expected));
+ }
+
+ private FeatureViewProto.FeatureViewSpec getFeatureViewSpec() {
+ return FeatureViewProto.FeatureViewSpec.newBuilder()
+ .setName("featuretable_1")
+ .addEntities("entity1")
+ .addEntities("entity2")
+ .addFeatures(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_1")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build())
+ .addFeatures(
+ FeatureProto.FeatureSpecV2.newBuilder()
+ .setName("feature_2")
+ .setValueType(ValueProto.ValueType.Enum.STRING)
+ .build())
+ .setTtl(Duration.newBuilder().setSeconds(120))
+ .build();
+ }
+
+ private GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2(
+ String projectName, List featureReferences) {
+ return GetOnlineFeaturesRequestV2.newBuilder()
+ .setProject(projectName)
+ .addAllFeatures(featureReferences)
+ .addEntityRows(
+ GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", createInt64Value(1))
+ .putFields("entity2", createStrValue("a")))
+ .addEntityRows(
+ GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", createInt64Value(2))
+ .putFields("entity2", createStrValue("b")))
+ .addFeatures(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_1")
+ .build())
+ .addFeatures(
+ ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretable_1")
+ .setName("feature_2")
+ .build())
+ .build();
+ }
+}
diff --git a/java/serving/src/test/java/feast/serving/util/DataGenerator.java b/java/serving/src/test/java/feast/serving/util/DataGenerator.java
new file mode 100644
index 00000000000..ab537fa6f9b
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/util/DataGenerator.java
@@ -0,0 +1,317 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.protobuf.Duration;
+import com.google.protobuf.Timestamp;
+import feast.proto.core.DataFormatProto.FileFormat;
+import feast.proto.core.DataFormatProto.FileFormat.ParquetFormat;
+import feast.proto.core.DataFormatProto.StreamFormat;
+import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat;
+import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat;
+import feast.proto.core.DataSourceProto.DataSource;
+import feast.proto.core.DataSourceProto.DataSource.FileOptions;
+import feast.proto.core.DataSourceProto.DataSource.KafkaOptions;
+import feast.proto.core.DataSourceProto.DataSource.KinesisOptions;
+import feast.proto.core.EntityProto;
+import feast.proto.core.FeatureProto;
+import feast.proto.core.FeatureProto.FeatureSpecV2;
+import feast.proto.core.FeatureTableProto.FeatureTableSpec;
+import feast.proto.core.StoreProto;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.types.ValueProto;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.tuple.Triple;
+
+public class DataGenerator {
+ // projectName, featureName, exclude
+ static Triple defaultSubscription = Triple.of("*", "*", false);
+
+ static StoreProto.Store defaultStore =
+ createStore(
+ "test-store", StoreProto.Store.StoreType.REDIS, ImmutableList.of(defaultSubscription));
+
+ public static Triple getDefaultSubscription() {
+ return defaultSubscription;
+ }
+
+ public static String valueToString(ValueProto.Value v) {
+ String stringRepr;
+ switch (v.getValCase()) {
+ case STRING_VAL:
+ stringRepr = v.getStringVal();
+ break;
+ case INT64_VAL:
+ stringRepr = String.valueOf(v.getInt64Val());
+ break;
+ case INT32_VAL:
+ stringRepr = String.valueOf(v.getInt32Val());
+ break;
+ case BYTES_VAL:
+ stringRepr = v.getBytesVal().toString();
+ break;
+ default:
+ throw new RuntimeException("Type is not supported to be entity");
+ }
+
+ return stringRepr;
+ }
+
+ public static StoreProto.Store getDefaultStore() {
+ return defaultStore;
+ }
+
+ public static StoreProto.Store createStore(
+ String name,
+ StoreProto.Store.StoreType type,
+ List> subscriptions) {
+ StoreProto.Store.Builder builder =
+ StoreProto.Store.newBuilder()
+ .addAllSubscriptions(
+ subscriptions.stream()
+ .map(
+ s ->
+ StoreProto.Store.Subscription.newBuilder()
+ .setProject(s.getLeft())
+ .setName(s.getMiddle())
+ .setExclude(s.getRight())
+ .build())
+ .collect(Collectors.toList()))
+ .setName(name)
+ .setType(type);
+
+ switch (type) {
+ case REDIS:
+ StoreProto.Store.RedisConfig redisConfig =
+ StoreProto.Store.RedisConfig.newBuilder().build();
+ return builder.setRedisConfig(redisConfig).build();
+ case REDIS_CLUSTER:
+ StoreProto.Store.RedisClusterConfig redisClusterConfig =
+ StoreProto.Store.RedisClusterConfig.newBuilder().build();
+ return builder.setRedisClusterConfig(redisClusterConfig).build();
+ default:
+ throw new RuntimeException("Unrecognized Store type");
+ }
+ }
+
+ public static EntityProto.EntitySpecV2 createEntitySpecV2(
+ String name,
+ String description,
+ ValueProto.ValueType.Enum valueType,
+ Map labels) {
+ return EntityProto.EntitySpecV2.newBuilder()
+ .setName(name)
+ .setDescription(description)
+ .setValueType(valueType)
+ .putAllLabels(labels)
+ .build();
+ }
+
+ public static FeatureProto.FeatureSpecV2 createFeatureSpecV2(
+ String name, ValueProto.ValueType.Enum valueType, Map labels) {
+ return FeatureProto.FeatureSpecV2.newBuilder()
+ .setName(name)
+ .setValueType(valueType)
+ .putAllLabels(labels)
+ .build();
+ }
+
+ // Create a Feature Table spec without DataSources configured.
+ public static FeatureTableSpec createFeatureTableSpec(
+ String name,
+ List entities,
+ Map features,
+ int maxAgeSecs,
+ Map labels) {
+
+ return FeatureTableSpec.newBuilder()
+ .setName(name)
+ .addAllEntities(entities)
+ .addAllFeatures(
+ features.entrySet().stream()
+ .map(
+ entry ->
+ FeatureSpecV2.newBuilder()
+ .setName(entry.getKey())
+ .setValueType(entry.getValue())
+ .putAllLabels(labels)
+ .build())
+ .collect(Collectors.toList()))
+ .setMaxAge(Duration.newBuilder().setSeconds(3600).build())
+ .setBatchSource(
+ DataSource.newBuilder()
+ .setEventTimestampColumn("ts")
+ .setType(DataSource.SourceType.BATCH_FILE)
+ .setFileOptions(
+ FileOptions.newBuilder()
+ .setFileFormat(
+ FileFormat.newBuilder()
+ .setParquetFormat(ParquetFormat.newBuilder().build())
+ .build())
+ .setFileUrl("/dev/null")
+ .build())
+ .build())
+ .putAllLabels(labels)
+ .build();
+ }
+
+ public static FeatureTableSpec createFeatureTableSpec(
+ String name,
+ List entities,
+ ImmutableMap features,
+ int maxAgeSecs,
+ Map labels) {
+
+ return FeatureTableSpec.newBuilder()
+ .setName(name)
+ .addAllEntities(entities)
+ .addAllFeatures(
+ features.entrySet().stream()
+ .map(
+ entry ->
+ FeatureSpecV2.newBuilder()
+ .setName(entry.getKey())
+ .setValueType(entry.getValue())
+ .putAllLabels(labels)
+ .build())
+ .collect(Collectors.toList()))
+ .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSecs).build())
+ .putAllLabels(labels)
+ .build();
+ }
+
+ public static DataSource createFileDataSourceSpec(
+ String fileURL, String timestampColumn, String datePartitionColumn) {
+ return DataSource.newBuilder()
+ .setType(DataSource.SourceType.BATCH_FILE)
+ .setFileOptions(
+ FileOptions.newBuilder()
+ .setFileFormat(createParquetFormat())
+ .setFileUrl(fileURL)
+ .build())
+ .setEventTimestampColumn(timestampColumn)
+ .setDatePartitionColumn(datePartitionColumn)
+ .build();
+ }
+
+ public static DataSource createBigQueryDataSourceSpec(
+ String bigQueryTableRef, String timestampColumn, String datePartitionColumn) {
+ return DataSource.newBuilder()
+ .setType(DataSource.SourceType.BATCH_BIGQUERY)
+ .setBigqueryOptions(
+ DataSource.BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build())
+ .setEventTimestampColumn(timestampColumn)
+ .setDatePartitionColumn(datePartitionColumn)
+ .build();
+ }
+
+ public static DataSource createKafkaDataSourceSpec(
+ String servers, String topic, String classPath, String timestampColumn) {
+ return DataSource.newBuilder()
+ .setType(DataSource.SourceType.STREAM_KAFKA)
+ .setKafkaOptions(
+ KafkaOptions.newBuilder()
+ .setTopic(topic)
+ .setBootstrapServers(servers)
+ .setMessageFormat(createProtoFormat("class.path"))
+ .build())
+ .setEventTimestampColumn(timestampColumn)
+ .build();
+ }
+
+ public static ValueProto.Value createEmptyValue() {
+ return ValueProto.Value.newBuilder().build();
+ }
+
+ public static ValueProto.Value createStrValue(String val) {
+ return ValueProto.Value.newBuilder().setStringVal(val).build();
+ }
+
+ public static ValueProto.Value createDoubleValue(double value) {
+ return ValueProto.Value.newBuilder().setDoubleVal(value).build();
+ }
+
+ public static ValueProto.Value createInt32Value(int value) {
+ return ValueProto.Value.newBuilder().setInt32Val(value).build();
+ }
+
+ public static ValueProto.Value createInt64Value(long value) {
+ return ValueProto.Value.newBuilder().setInt64Val(value).build();
+ }
+
+ public static ServingAPIProto.FeatureReferenceV2 createFeatureReference(
+ String featureTableName, String featureName) {
+ return ServingAPIProto.FeatureReferenceV2.newBuilder()
+ .setFeatureTable(featureTableName)
+ .setName(featureName)
+ .build();
+ }
+
+ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityRow(
+ String entityName, ValueProto.Value entityValue, long seconds) {
+ return ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(seconds))
+ .putFields(entityName, entityValue)
+ .build();
+ }
+
+ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createCompoundEntityRow(
+ ImmutableMap entityNameValues, long seconds) {
+ ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.Builder entityRow =
+ ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+ .setTimestamp(Timestamp.newBuilder().setSeconds(seconds));
+
+ entityNameValues.entrySet().stream()
+ .forEach(entry -> entityRow.putFields(entry.getKey(), entry.getValue()));
+
+ return entityRow.build();
+ }
+
+ public static DataSource createKinesisDataSourceSpec(
+ String region, String streamName, String classPath, String timestampColumn) {
+ return DataSource.newBuilder()
+ .setType(DataSource.SourceType.STREAM_KINESIS)
+ .setKinesisOptions(
+ KinesisOptions.newBuilder()
+ .setRegion("ap-nowhere1")
+ .setStreamName("stream")
+ .setRecordFormat(createProtoFormat(classPath))
+ .build())
+ .setEventTimestampColumn(timestampColumn)
+ .build();
+ }
+
+ public static FileFormat createParquetFormat() {
+ return FileFormat.newBuilder().setParquetFormat(ParquetFormat.getDefaultInstance()).build();
+ }
+
+ public static StreamFormat createAvroFormat(String schemaJSON) {
+ return StreamFormat.newBuilder()
+ .setAvroFormat(AvroFormat.newBuilder().setSchemaJson(schemaJSON).build())
+ .build();
+ }
+
+ public static StreamFormat createProtoFormat(String classPath) {
+ return StreamFormat.newBuilder()
+ .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build())
+ .build();
+ }
+}
diff --git a/java/serving/src/test/java/feast/serving/util/RequestHelperTest.java b/java/serving/src/test/java/feast/serving/util/RequestHelperTest.java
new file mode 100644
index 00000000000..140d46cd569
--- /dev/null
+++ b/java/serving/src/test/java/feast/serving/util/RequestHelperTest.java
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
+import org.junit.Test;
+
+public class RequestHelperTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldErrorIfEntityRowEmpty() {
+ FeatureReferenceV2 featureReference =
+ FeatureReferenceV2.newBuilder()
+ .setFeatureTable("featuretablename")
+ .setName("featurename")
+ .build();
+ GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 =
+ GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build();
+ RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldErrorIfFeatureReferenceTableEmpty() {
+ FeatureReferenceV2 featureReference =
+ FeatureReferenceV2.newBuilder().setName("featurename").build();
+ GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 =
+ GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build();
+ RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldErrorIfFeatureReferenceNameEmpty() {
+ FeatureReferenceV2 featureReference =
+ FeatureReferenceV2.newBuilder().setFeatureTable("featuretablename").build();
+ GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 =
+ GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build();
+ RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2);
+ }
+}
diff --git a/java/serving/src/test/resources/application-it.properties b/java/serving/src/test/resources/application-it.properties
new file mode 100644
index 00000000000..6b56408d439
--- /dev/null
+++ b/java/serving/src/test/resources/application-it.properties
@@ -0,0 +1,14 @@
+#
+# Copyright 2018 The Feast Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml
new file mode 100644
index 00000000000..08a50233df1
--- /dev/null
+++ b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml
@@ -0,0 +1,12 @@
+version: '3'
+
+services:
+ redis:
+ image: redis:5-alpine
+ ports:
+ - "6379:6379"
+ materialize:
+ build: feast10
+ links:
+ - redis
+
diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile
new file mode 100644
index 00000000000..bde9f11592f
--- /dev/null
+++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3.7
+
+WORKDIR /usr/src/
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD [ "python", "./materialize.py" ]
diff --git a/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml
new file mode 100644
index 00000000000..7554725004f
--- /dev/null
+++ b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml
@@ -0,0 +1,10 @@
+project: feast_project
+registry: registry.db
+provider: local
+online_store:
+ type: redis
+ connection_string: "redis:6379"
+offline_store: {}
+flags:
+ alpha_features: true
+ on_demand_transforms: true
diff --git a/java/serving/src/test/resources/docker-compose/feast10/materialize.py b/java/serving/src/test/resources/docker-compose/feast10/materialize.py
new file mode 100644
index 00000000000..c347728c68b
--- /dev/null
+++ b/java/serving/src/test/resources/docker-compose/feast10/materialize.py
@@ -0,0 +1,64 @@
+# This is an example feature definition file
+
+import pandas as pd
+import numpy as np
+
+from google.protobuf.duration_pb2 import Duration
+
+from datetime import datetime, timedelta
+from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, FeatureStore
+
+print("Running materialize.py")
+
+# Fill our temporary data source
+start = datetime.now() - timedelta(days=10)
+
+df = pd.DataFrame()
+df["driver_id"] = np.arange(1000, 1010)
+df["created"] = datetime.now()
+df["conv_rate"] = np.arange(0, 1, 0.1)
+df["acc_rate"] = np.arange(0.5, 1, 0.05)
+df["avg_daily_trips"] = np.arange(0, 1000, 100)
+
+# some of rows are beyond 7 days to test OUTSIDE_MAX_AGE status
+df["event_timestamp"] = start + pd.Series(np.arange(0, 10)).map(lambda days: timedelta(days=days))
+
+df.to_parquet("driver_stats.parquet")
+
+# Read data from parquet files. Parquet is convenient for local development mode. For
+# production, you can use your favorite DWH, such as BigQuery. See Feast documentation
+# for more info.
+file_path = "driver_stats.parquet"
+driver_hourly_stats = FileSource(
+ path=file_path,
+ event_timestamp_column="event_timestamp",
+ created_timestamp_column="created",
+)
+
+# Define an entity for the driver. You can think of entity as a primary key used to
+# fetch features.
+driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",)
+
+# Our parquet files contain sample data that includes a driver_id column, timestamps and
+# three feature column. Here we define a Feature View that will allow us to serve this
+# data to our model online.
+driver_hourly_stats_view = FeatureView(
+ name="driver_hourly_stats",
+ entities=["driver_id"],
+ ttl=Duration(seconds=86400 * 7),
+ features=[
+ Feature(name="conv_rate", dtype=ValueType.DOUBLE),
+ Feature(name="acc_rate", dtype=ValueType.FLOAT),
+ Feature(name="avg_daily_trips", dtype=ValueType.INT64),
+ ],
+ online=True,
+ batch_source=driver_hourly_stats,
+ tags={},
+)
+
+fs = FeatureStore(".")
+fs.apply([driver_hourly_stats_view, driver])
+
+now = datetime.now()
+fs.materialize(start, now)
+print("Materialization finished")
diff --git a/java/serving/src/test/resources/docker-compose/feast10/registry.db b/java/serving/src/test/resources/docker-compose/feast10/registry.db
new file mode 100644
index 00000000000..b9a19475af0
Binary files /dev/null and b/java/serving/src/test/resources/docker-compose/feast10/registry.db differ
diff --git a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt
new file mode 100644
index 00000000000..447f126392b
--- /dev/null
+++ b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt
@@ -0,0 +1,8 @@
+feast[redis]>=0.13,<1
+
+# for source generation
+pyarrow==6.0.0
+
+# temp fixes
+proto-plus
+Jinja2>=2.0.0
\ No newline at end of file
diff --git a/java/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/java/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 00000000000..ca6ee9cea8e
--- /dev/null
+++ b/java/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file
diff --git a/java/storage/api/pom.xml b/java/storage/api/pom.xml
new file mode 100644
index 00000000000..6bce6cec4eb
--- /dev/null
+++ b/java/storage/api/pom.xml
@@ -0,0 +1,78 @@
+
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../..
+
+
+ 4.0.0
+ feast-storage-api
+
+ Feast Storage API
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+
+ javax.annotation
+
+
+
+
+
+
+
+
+
+ dev.feast
+ datatypes-java
+ ${project.version}
+
+
+
+
+
+
+
+
+
+ com.google.auto.value
+ auto-value-annotations
+ 1.6.6
+
+
+
+ com.google.auto.value
+ auto-value
+ 1.6.6
+ provided
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.9
+
+
+
+ org.apache.avro
+ avro
+ 1.10.2
+
+
+
+ junit
+ junit
+ 4.13.2
+ test
+
+
+
+
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java b/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java
new file mode 100644
index 00000000000..96f19ccdbe1
--- /dev/null
+++ b/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java
@@ -0,0 +1,171 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retriever;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.types.ValueProto;
+import java.nio.ByteBuffer;
+import java.util.stream.Collectors;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.util.Utf8;
+
+public class AvroFeature implements Feature {
+ private final ServingAPIProto.FeatureReferenceV2 featureReference;
+
+ private final Timestamp eventTimestamp;
+
+ private final Object featureValue;
+
+ public AvroFeature(
+ ServingAPIProto.FeatureReferenceV2 featureReference,
+ Timestamp eventTimestamp,
+ Object featureValue) {
+ this.featureReference = featureReference;
+ this.eventTimestamp = eventTimestamp;
+ this.featureValue = featureValue;
+ }
+
+ /**
+ * Casts feature value of Object type based on Feast valueType. Empty object i.e new Object() is
+ * interpreted as VAL_NOT_SET Feast valueType.
+ *
+ * @param valueType Feast valueType of feature as specified in FeatureSpec
+ * @return ValueProto.Value representation of feature
+ */
+ @Override
+ public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) {
+ ValueProto.Value finalValue;
+
+ try {
+ switch (valueType) {
+ case STRING:
+ finalValue =
+ ValueProto.Value.newBuilder().setStringVal(((Utf8) featureValue).toString()).build();
+ break;
+ case INT32:
+ finalValue = ValueProto.Value.newBuilder().setInt32Val((Integer) featureValue).build();
+ break;
+ case INT64:
+ finalValue = ValueProto.Value.newBuilder().setInt64Val((Long) featureValue).build();
+ break;
+ case DOUBLE:
+ finalValue = ValueProto.Value.newBuilder().setDoubleVal((Double) featureValue).build();
+ break;
+ case FLOAT:
+ finalValue = ValueProto.Value.newBuilder().setFloatVal((Float) featureValue).build();
+ break;
+ case BYTES:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setBytesVal(ByteString.copyFrom(((ByteBuffer) featureValue).array()))
+ .build();
+ break;
+ case BOOL:
+ finalValue = ValueProto.Value.newBuilder().setBoolVal((Boolean) featureValue).build();
+ break;
+ case STRING_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setStringListVal(
+ ValueProto.StringList.newBuilder()
+ .addAllVal(
+ ((GenericData.Array) featureValue)
+ .stream().map(Utf8::toString).collect(Collectors.toList()))
+ .build())
+ .build();
+ break;
+ case INT64_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setInt64ListVal(
+ ValueProto.Int64List.newBuilder()
+ .addAllVal(((GenericData.Array) featureValue))
+ .build())
+ .build();
+ break;
+ case INT32_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setInt32ListVal(
+ ValueProto.Int32List.newBuilder()
+ .addAllVal(((GenericData.Array) featureValue))
+ .build())
+ .build();
+ break;
+ case FLOAT_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setFloatListVal(
+ ValueProto.FloatList.newBuilder()
+ .addAllVal(((GenericData.Array) featureValue))
+ .build())
+ .build();
+ break;
+ case DOUBLE_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setDoubleListVal(
+ ValueProto.DoubleList.newBuilder()
+ .addAllVal(((GenericData.Array) featureValue))
+ .build())
+ .build();
+ break;
+ case BOOL_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setBoolListVal(
+ ValueProto.BoolList.newBuilder()
+ .addAllVal(((GenericData.Array) featureValue))
+ .build())
+ .build();
+ break;
+ case BYTES_LIST:
+ finalValue =
+ ValueProto.Value.newBuilder()
+ .setBytesListVal(
+ ValueProto.BytesList.newBuilder()
+ .addAllVal(
+ ((GenericData.Array) featureValue)
+ .stream()
+ .map(byteBuffer -> ByteString.copyFrom(byteBuffer.array()))
+ .collect(Collectors.toList()))
+ .build())
+ .build();
+ break;
+ default:
+ throw new RuntimeException("FeatureType is not supported");
+ }
+ } catch (ClassCastException e) {
+ // Feature type has changed
+ finalValue = ValueProto.Value.newBuilder().build();
+ }
+
+ return finalValue;
+ }
+
+ @Override
+ public ServingAPIProto.FeatureReferenceV2 getFeatureReference() {
+ return this.featureReference;
+ }
+
+ @Override
+ public Timestamp getEventTimestamp() {
+ return this.eventTimestamp;
+ }
+}
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java b/java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java
new file mode 100644
index 00000000000..92ae1f31fb1
--- /dev/null
+++ b/java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java
@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retriever;
+
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.types.ValueProto;
+import feast.proto.types.ValueProto.Value;
+import java.util.HashMap;
+
+public interface Feature {
+
+ HashMap TYPE_TO_VAL_CASE =
+ new HashMap() {
+ {
+ put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL);
+ put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL);
+ put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL);
+ put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL);
+ put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL);
+ put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL);
+ put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL);
+ put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL);
+ put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL);
+ put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL);
+ put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL);
+ put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL);
+ put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL);
+ put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL);
+ }
+ };
+
+ Value getFeatureValue(ValueProto.ValueType.Enum valueType);
+
+ FeatureReferenceV2 getFeatureReference();
+
+ Timestamp getEventTimestamp();
+}
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java b/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java
new file mode 100644
index 00000000000..6188a270c40
--- /dev/null
+++ b/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retriever;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSet;
+import feast.proto.core.FeatureTableProto.FeatureTableSpec;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@AutoValue
+public abstract class FeatureTableRequest {
+ public abstract FeatureTableSpec getSpec();
+
+ public abstract ImmutableSet getFeatureReferences();
+
+ public static Builder newBuilder() {
+ return new AutoValue_FeatureTableRequest.Builder();
+ }
+
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder setSpec(FeatureTableSpec spec);
+
+ abstract ImmutableSet.Builder featureReferencesBuilder();
+
+ public Builder addAllFeatureReferences(List featureReferenceList) {
+ featureReferencesBuilder().addAll(featureReferenceList);
+ return this;
+ }
+
+ public Builder addFeatureReference(FeatureReferenceV2 featureReference) {
+ featureReferencesBuilder().add(featureReference);
+ return this;
+ }
+
+ public abstract FeatureTableRequest build();
+ }
+
+ public Map getFeatureRefsByName() {
+ return getFeatureReferences().stream()
+ .collect(
+ Collectors.toMap(FeatureReferenceV2::getName, featureReference -> featureReference));
+ }
+}
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java
new file mode 100644
index 00000000000..a49ab3fccd0
--- /dev/null
+++ b/java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retriever;
+
+import feast.proto.serving.ServingAPIProto;
+import java.util.List;
+
+public interface OnlineRetrieverV2 {
+ /**
+ * Get online features for the given entity rows using data retrieved from the Feature references
+ * specified in FeatureTable request.
+ *
+ * Each {@link Feature} optional in the returned list then corresponds to an {@link
+ * ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} provided by the user. If feature for a
+ * given entity row is not found, will return an empty optional instead. The no. of {@link
+ * Feature} returned should match the no. of given {@link
+ * ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s
+ *
+ * @param project name of project to request features from.
+ * @param entityRows list of entity rows to request features for.
+ * @param featureReferences specifies the FeatureTable to retrieve data from
+ * @param entityNames name of entities
+ * @return list of {@link Feature}s corresponding to data retrieved for each entity row from
+ * FeatureTable specified in FeatureTable request.
+ */
+ List> getOnlineFeatures(
+ String project,
+ List entityRows,
+ List featureReferences,
+ List entityNames);
+}
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java b/java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java
new file mode 100644
index 00000000000..09f6b75f49c
--- /dev/null
+++ b/java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retriever;
+
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto;
+import feast.proto.types.ValueProto;
+
+public class ProtoFeature implements Feature {
+ private final ServingAPIProto.FeatureReferenceV2 featureReference;
+
+ private final Timestamp eventTimestamp;
+
+ private final ValueProto.Value featureValue;
+
+ public ProtoFeature(
+ ServingAPIProto.FeatureReferenceV2 featureReference,
+ Timestamp eventTimestamp,
+ ValueProto.Value featureValue) {
+ this.featureReference = featureReference;
+ this.eventTimestamp = eventTimestamp;
+ this.featureValue = featureValue;
+ }
+
+ /**
+ * Returns Feast valueType if type matches, otherwise null.
+ *
+ * @param valueType Feast valueType of feature as specified in FeatureSpec
+ * @return ValueProto.Value representation of feature
+ */
+ @Override
+ public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) {
+ if (TYPE_TO_VAL_CASE.get(valueType) != this.featureValue.getValCase()) {
+ return null;
+ }
+
+ return this.featureValue;
+ }
+
+ @Override
+ public ServingAPIProto.FeatureReferenceV2 getFeatureReference() {
+ return this.featureReference;
+ }
+
+ @Override
+ public Timestamp getEventTimestamp() {
+ return this.eventTimestamp;
+ }
+}
diff --git a/java/storage/connectors/bigtable/pom.xml b/java/storage/connectors/bigtable/pom.xml
new file mode 100644
index 00000000000..81cd4507450
--- /dev/null
+++ b/java/storage/connectors/bigtable/pom.xml
@@ -0,0 +1,45 @@
+
+
+
+ dev.feast
+ feast-storage-connectors
+ ${revision}
+
+
+ 4.0.0
+ feast-storage-connector-bigtable
+
+
+ 11
+ 11
+
+
+
+
+ com.google.cloud
+ google-cloud-bigtable
+ 1.21.2
+
+
+
+
+ org.apache.avro
+ avro
+ 1.10.2
+
+
+
+ dev.feast
+ feast-storage-connector-sstable
+ ${project.version}
+
+
+
+ com.google.guava
+ guava
+
+
+
+
\ No newline at end of file
diff --git a/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java
new file mode 100644
index 00000000000..6e67782b2ac
--- /dev/null
+++ b/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java
@@ -0,0 +1,211 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigtable.retriever;
+
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.models.Filters;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowCell;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Timestamp;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow;
+import feast.storage.api.retriever.AvroFeature;
+import feast.storage.api.retriever.Feature;
+import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever;
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.*;
+
+public class BigTableOnlineRetriever implements SSTableOnlineRetriever {
+
+ private BigtableDataClient client;
+ private BigTableSchemaRegistry schemaRegistry;
+
+ public BigTableOnlineRetriever(BigtableDataClient client) {
+ this.client = client;
+ this.schemaRegistry = new BigTableSchemaRegistry(client);
+ }
+
+ /**
+ * Generate BigTable key in the form of entity values joined by #.
+ *
+ * @param entityRow Single EntityRow representation in feature retrieval call
+ * @param entityNames List of entities related to feature references in retrieval call
+ * @return BigTable key for retrieval
+ */
+ @Override
+ public ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) {
+ return ByteString.copyFrom(
+ entityNames.stream()
+ .map(entity -> entityRow.getFieldsMap().get(entity))
+ .map(this::valueToString)
+ .collect(Collectors.joining("#"))
+ .getBytes());
+ }
+
+ /**
+ * Converts rowCell feature value into @NativeFeature type.
+ *
+ * @param tableName Name of BigTable table
+ * @param rowKeys List of keys of rows to retrieve
+ * @param rows Map of rowKey to Row related to it
+ * @param featureReferences List of feature references
+ * @return List of List of Features associated with respective rowKey
+ */
+ @Override
+ public List> convertRowToFeature(
+ String tableName,
+ List rowKeys,
+ Map rows,
+ List featureReferences) {
+
+ BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null);
+
+ return rowKeys.stream()
+ .map(
+ rowKey -> {
+ if (!rows.containsKey(rowKey)) {
+ return Collections.emptyList();
+ } else {
+ Row row = rows.get(rowKey);
+ return featureReferences.stream()
+ .map(FeatureReferenceV2::getFeatureTable)
+ .distinct()
+ .map(cf -> row.getCells(cf, ""))
+ .filter(ls -> !ls.isEmpty())
+ .flatMap(
+ rowCells -> {
+ RowCell rowCell = rowCells.get(0); // Latest cell
+ String family = rowCell.getFamily();
+ ByteString value = rowCell.getValue();
+
+ List features;
+ List localFeatureReferences =
+ featureReferences.stream()
+ .filter(
+ featureReference ->
+ featureReference.getFeatureTable().equals(family))
+ .collect(Collectors.toList());
+
+ try {
+ features =
+ decodeFeatures(
+ tableName,
+ value,
+ localFeatureReferences,
+ reusedDecoder,
+ rowCell.getTimestamp());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to decode features from BigTable");
+ }
+
+ return features.stream();
+ })
+ .collect(Collectors.toList());
+ }
+ })
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Retrieve rows for each row entity key by generating BigTable rowQuery with filters based on
+ * column families.
+ *
+ * @param tableName Name of BigTable table
+ * @param rowKeys List of keys of rows to retrieve
+ * @param columnFamilies List of FeatureTable names
+ * @return Map of retrieved features for each rowKey
+ */
+ @Override
+ public Map getFeaturesFromSSTable(
+ String tableName, List rowKeys, List columnFamilies) {
+ Query rowQuery = Query.create(tableName);
+ Filters.InterleaveFilter familyFilter = Filters.FILTERS.interleave();
+ columnFamilies.forEach(cf -> familyFilter.filter(Filters.FILTERS.family().exactMatch(cf)));
+
+ for (ByteString rowKey : rowKeys) {
+ rowQuery.rowKey(rowKey);
+ }
+
+ return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false)
+ .collect(Collectors.toMap(Row::getKey, Function.identity()));
+ }
+
+ /**
+ * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is
+ * returned when null is retrieved from BigTable RowCell.
+ *
+ * @param tableName Name of BigTable table
+ * @param value Value of BigTable cell where first 4 bytes represent the schema reference and
+ * remaining bytes represent avro-serialized features
+ * @param featureReferences List of feature references
+ * @param reusedDecoder Decoder for decoding feature values
+ * @param timestamp Timestamp of rowCell
+ * @return @NativeFeature with retrieved value stored in BigTable RowCell
+ * @throws IOException
+ */
+ private List decodeFeatures(
+ String tableName,
+ ByteString value,
+ List featureReferences,
+ BinaryDecoder reusedDecoder,
+ long timestamp)
+ throws IOException {
+ ByteString schemaReferenceBytes = value.substring(0, 4);
+ byte[] featureValueBytes = value.substring(4).toByteArray();
+
+ BigTableSchemaRegistry.SchemaReference schemaReference =
+ new BigTableSchemaRegistry.SchemaReference(tableName, schemaReferenceBytes);
+
+ GenericDatumReader reader = schemaRegistry.getReader(schemaReference);
+
+ reusedDecoder = DecoderFactory.get().binaryDecoder(featureValueBytes, reusedDecoder);
+ GenericRecord record = reader.read(null, reusedDecoder);
+
+ return featureReferences.stream()
+ .map(
+ featureReference -> {
+ Object featureValue;
+ try {
+ featureValue = record.get(featureReference.getName());
+ } catch (AvroRuntimeException e) {
+ // Feature is not found in schema
+ return null;
+ }
+ if (featureValue != null) {
+ return new AvroFeature(
+ featureReference,
+ Timestamp.newBuilder().setSeconds(timestamp / 1000).build(),
+ featureValue);
+ }
+ return new AvroFeature(
+ featureReference,
+ Timestamp.newBuilder().setSeconds(timestamp / 1000).build(),
+ new Object());
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+}
diff --git a/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java b/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java
new file mode 100644
index 00000000000..64989f0f8dd
--- /dev/null
+++ b/java/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java
@@ -0,0 +1,107 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigtable.retriever;
+
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.models.Filters;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowCell;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+import java.util.concurrent.ExecutionException;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+
+public class BigTableSchemaRegistry {
+ private final BigtableDataClient client;
+ private final LoadingCache> cache;
+
+ private static String COLUMN_FAMILY = "metadata";
+ private static String QUALIFIER = "avro";
+ private static String KEY_PREFIX = "schema#";
+
+ public static class SchemaReference {
+ private final String tableName;
+ private final ByteString schemaHash;
+
+ public SchemaReference(String tableName, ByteString schemaHash) {
+ this.tableName = tableName;
+ this.schemaHash = schemaHash;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public ByteString getSchemaHash() {
+ return schemaHash;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = tableName.hashCode();
+ result = 31 * result + schemaHash.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SchemaReference that = (SchemaReference) o;
+
+ if (!tableName.equals(that.tableName)) return false;
+ return schemaHash.equals(that.schemaHash);
+ }
+ }
+
+ public BigTableSchemaRegistry(BigtableDataClient client) {
+ this.client = client;
+
+ CacheLoader> schemaCacheLoader =
+ CacheLoader.from(this::loadReader);
+
+ cache = CacheBuilder.newBuilder().build(schemaCacheLoader);
+ }
+
+ public GenericDatumReader getReader(SchemaReference reference) {
+ GenericDatumReader