job = jobInfoRepository.findById(id);
// if (!job.isPresent()) {
- // throw new RetrievalException(Strings.lenientFormat("Unable to retrieve job with id %s",
+ // throw new RetrievalException(String.format("Unable to retrieve job with id %s",
// id));
// }
// JobDetail.Builder jobDetailBuilder = job.get().getJobDetail().toBuilder();
diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java
index 4ea2d288f2e..ffa1aa08e81 100644
--- a/core/src/main/java/feast/core/service/SpecService.java
+++ b/core/src/main/java/feast/core/service/SpecService.java
@@ -82,7 +82,7 @@ public SpecService(
* required. If the version is provided then it will be used for the lookup. If the version is
* omitted then the latest version will be returned.
*
- * @param GetFeatureSetRequest containing the name and version of the feature set
+ * @param request containing the name and version of the feature set
* @return GetFeatureSetResponse containing a single feature set
*/
public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
@@ -110,8 +110,9 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
if (featureSet == null) {
throw io.grpc.Status.NOT_FOUND
- .withDescription(String.format("Feature set with name \"%s\" could not be found.",
- request.getName()))
+ .withDescription(
+ String.format(
+ "Feature set with name \"%s\" could not be found.", request.getName()))
.asRuntimeException();
}
} else {
@@ -121,13 +122,14 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
if (featureSet == null) {
throw io.grpc.Status.NOT_FOUND
- .withDescription(String.format("Feature set with name \"%s\" and version \"%s\" could "
- + "not be found.", request.getName(), request.getVersion()))
+ .withDescription(
+ String.format(
+ "Feature set with name \"%s\" and version \"%s\" could " + "not be found.",
+ request.getName(), request.getVersion()))
.asRuntimeException();
}
}
-
// Only a single item in list, return successfully
return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSet.toProto()).build();
}
@@ -141,7 +143,7 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
*
* The version filter is optional; If not provided, this method will return all featureSet
* versions of the featureSet name provided. Valid version filters should optionally contain a
- * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1
+ * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1
*
* @param filter filter containing the desired featureSet name and version filter
* @return ListFeatureSetsResponse with list of featureSets found matching the filter
@@ -154,7 +156,9 @@ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter fil
if (name.equals("")) {
featureSets = featureSetRepository.findAllByOrderByNameAscVersionAsc();
} else {
- featureSets = featureSetRepository.findByNameWithWildcardOrderByNameAscVersionAsc(name.replace('*', '%'));
+ featureSets =
+ featureSetRepository.findByNameWithWildcardOrderByNameAscVersionAsc(
+ name.replace('*', '%'));
featureSets =
featureSets.stream()
.filter(getVersionFilter(filter.getFeatureSetVersion()))
diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java
index ef27332ac0f..99c5d73ba78 100644
--- a/core/src/main/java/feast/core/util/PackageUtil.java
+++ b/core/src/main/java/feast/core/util/PackageUtil.java
@@ -44,13 +44,14 @@ public class PackageUtil {
* points to the resource location. Note that the extraction process can take several minutes to
* complete.
*
- *
One use case of this function is to detect the class path of resources to stage when
- * using Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be
- * handled by default in Apache Beam.
+ *
One use case of this function is to detect the class path of resources to stage when using
+ * Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be handled by
+ * default in Apache Beam.
*
*
- * @code
+ *
* URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
+ *
* String resolvedPath = resolveSpringBootPackageClasspath(url);
* // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
* // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.
diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java
index e01a5511359..5fe69819476 100644
--- a/core/src/main/java/feast/core/util/TypeConversion.java
+++ b/core/src/main/java/feast/core/util/TypeConversion.java
@@ -16,7 +16,6 @@
*/
package feast.core.util;
-import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Type;
@@ -85,7 +84,7 @@ public static String convertMapToJsonString(Map map) {
public static String[] convertMapToArgs(Map map) {
List args = new ArrayList<>();
for (Entry arg : map.entrySet()) {
- args.add(Strings.lenientFormat("--%s=%s", arg.getKey(), arg.getValue()));
+ args.add(String.format("--%s=%s", arg.getKey(), arg.getValue()));
}
return args.toArray(new String[] {});
}
diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/core/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/core/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java
index a11adf022b9..6d112676350 100644
--- a/core/src/test/java/feast/core/service/SpecServiceTest.java
+++ b/core/src/test/java/feast/core/service/SpecServiceTest.java
@@ -68,14 +68,11 @@
public class SpecServiceTest {
- @Mock
- private FeatureSetRepository featureSetRepository;
+ @Mock private FeatureSetRepository featureSetRepository;
- @Mock
- private StoreRepository storeRepository;
+ @Mock private StoreRepository storeRepository;
- @Rule
- public final ExpectedException expectedException = ExpectedException.none();
+ @Rule public final ExpectedException expectedException = ExpectedException.none();
private SpecService specService;
private List featureSets;
@@ -102,11 +99,12 @@ public void setUp() {
Field f3f1 = new Field("f3", "f3f1", Enum.INT64);
Field f3f2 = new Field("f3", "f3f2", Enum.INT64);
Field f3e1 = new Field("f3", "f3e1", Enum.STRING);
- FeatureSet featureSet3v1 = new FeatureSet(
- "f3", 1, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource);
+ FeatureSet featureSet3v1 =
+ new FeatureSet(
+ "f3", 1, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource);
- featureSets = Arrays
- .asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1);
+ featureSets =
+ Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1);
when(featureSetRepository.findAll()).thenReturn(featureSets);
when(featureSetRepository.findAllByOrderByNameAscVersionAsc()).thenReturn(featureSets);
when(featureSetRepository.findByName("f1")).thenReturn(featureSets.subList(0, 3));
@@ -347,7 +345,6 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(expected));
}
-
@Test
public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
throws InvalidProtocolBufferException {
@@ -355,20 +352,21 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
Field f3f1 = new Field("f3", "f3f1", Enum.INT64);
Field f3f2 = new Field("f3", "f3f2", Enum.INT64);
Field f3e1 = new Field("f3", "f3e1", Enum.STRING);
- FeatureSetProto.FeatureSetSpec incomingFeatureSet = (new FeatureSet(
- "f3", 5, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource)).toProto();
+ FeatureSetProto.FeatureSetSpec incomingFeatureSet =
+ (new FeatureSet(
+ "f3", 5, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource))
+ .toProto();
FeatureSetSpec expected = incomingFeatureSet;
ApplyFeatureSetResponse applyFeatureSetResponse =
specService.applyFeatureSet(incomingFeatureSet);
assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.NO_CHANGE));
assertThat(applyFeatureSetResponse.getFeatureSet().getMaxAge(), equalTo(expected.getMaxAge()));
- assertThat(applyFeatureSetResponse.getFeatureSet().getEntities(0),
- equalTo(expected.getEntities(0)));
+ assertThat(
+ applyFeatureSetResponse.getFeatureSet().getEntities(0), equalTo(expected.getEntities(0)));
assertThat(applyFeatureSetResponse.getFeatureSet().getName(), equalTo(expected.getName()));
}
-
@Test
public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException {
when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0)));
diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java
index 774e58c7a87..3bf09dd474f 100644
--- a/core/src/test/java/feast/core/validators/MatchersTest.java
+++ b/core/src/test/java/feast/core/validators/MatchersTest.java
@@ -19,7 +19,6 @@
import static feast.core.validators.Matchers.checkLowerSnakeCase;
import static feast.core.validators.Matchers.checkUpperSnakeCase;
-import com.google.common.base.Strings;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -43,7 +42,7 @@ public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() {
public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() {
exception.expect(IllegalArgumentException.class);
exception.expectMessage(
- Strings.lenientFormat(
+ String.format(
"invalid value for field %s: %s",
"someField",
"argument must be in upper snake case, and cannot include any special characters."));
@@ -61,7 +60,7 @@ public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() {
public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() {
exception.expect(IllegalArgumentException.class);
exception.expectMessage(
- Strings.lenientFormat(
+ String.format(
"invalid value for field %s: %s",
"someField",
"argument must be in lower snake case, and cannot include any special characters."));
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
new file mode 100644
index 00000000000..a062144ff3f
--- /dev/null
+++ b/datatypes/java/README.md
@@ -0,0 +1,43 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+
+ dev.feast
+ datatypes-java
+ 0.3.6-SNAPSHOT
+
+```
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Publishing
+----------
+
+TODO: this module should be published to Maven Central upon Feast releases—this
+needs to be set up in POM configuration and release automation.
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
new file mode 100644
index 00000000000..415d204293a
--- /dev/null
+++ b/datatypes/java/pom.xml
@@ -0,0 +1,110 @@
+
+
+
+ 4.0.0
+
+ Feast Data Types for Java
+
+ Data types and service contracts used throughout Feast components and
+ their interchanges. These are generated from Protocol Buffers and gRPC
+ definitions included in the package.
+
+ datatypes-java
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../..
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+
+ javax.annotation
+
+
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+
+ true
+
+ com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+
+ grpc-java
+
+ io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+
+
+
+
+
+ compile
+ compile-custom
+ test-compile
+
+
+
+
+
+
+
+
+
+
+ com.google.guava
+ guava
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+
+ io.grpc
+ grpc-core
+
+
+ io.grpc
+ grpc-protobuf
+
+
+ io.grpc
+ grpc-services
+
+
+ io.grpc
+ grpc-stub
+
+
+
+ javax.annotation
+ javax.annotation-api
+
+
+
diff --git a/sdk/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast
similarity index 100%
rename from sdk/java/src/main/proto/feast
rename to datatypes/java/src/main/proto/feast
diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party
new file mode 120000
index 00000000000..f015f8477d1
--- /dev/null
+++ b/datatypes/java/src/main/proto/third_party
@@ -0,0 +1 @@
+../../../../../protos/third_party
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml
index b312a40692c..c5ccf108698 100644
--- a/infra/charts/feast/charts/feast-serving/values.yaml
+++ b/infra/charts/feast/charts/feast-serving/values.yaml
@@ -56,6 +56,15 @@ application.yaml:
config-path: /etc/feast/feast-serving/store.yaml
redis-pool-max-size: 128
redis-pool-max-idle: 64
+ cassandra-pool-core-local-connections: 1
+ cassandra-pool-max-local-connections: 1
+ cassandra-pool-core-remote-connections: 1
+ cassandra-pool-max-remote-connections: 1
+ cassandra-pool-max-requests-local-connection: 32768
+ cassandra-pool-max-requests-remote-connection: 2048
+ cassandra-pool-new-local-connection-threshold: 30000
+ cassandra-pool-new-remote-connection-threshold: 400
+ cassandra-pool-timeout-millis: 0
jobs:
staging-location: ""
store-type: ""
diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile
index c4cfe34b71b..91ef030dc90 100644
--- a/infra/docker/core/Dockerfile
+++ b/infra/docker/core/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
# the existing .m2 directory to $FEAST_REPO_ROOT/.m2
#
ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \
+RUN mvn --also-make --projects core,ingestion \
-DskipTests=true --batch-mode package
#
# Unpack the jar and copy the files into production Docker image
diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
index 3517183d782..5605c8846de 100644
--- a/infra/docker/serving/Dockerfile
+++ b/infra/docker/serving/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
# the existing .m2 directory to $FEAST_REPO_ROOT/.m2
#
ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+RUN mvn --also-make --projects serving \
-DskipTests=true --batch-mode package
# ============================================================
diff --git a/.prow/scripts/download-maven-cache.sh b/infra/scripts/download-maven-cache.sh
similarity index 100%
rename from .prow/scripts/download-maven-cache.sh
rename to infra/scripts/download-maven-cache.sh
diff --git a/.prow/scripts/install-google-cloud-sdk.sh b/infra/scripts/install-google-cloud-sdk.sh
similarity index 100%
rename from .prow/scripts/install-google-cloud-sdk.sh
rename to infra/scripts/install-google-cloud-sdk.sh
diff --git a/.prow/scripts/publish-docker-image.sh b/infra/scripts/publish-docker-image.sh
similarity index 100%
rename from .prow/scripts/publish-docker-image.sh
rename to infra/scripts/publish-docker-image.sh
diff --git a/infra/scripts/publish-java-sdk.sh b/infra/scripts/publish-java-sdk.sh
new file mode 100755
index 00000000000..91123c8d4ee
--- /dev/null
+++ b/infra/scripts/publish-java-sdk.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+GPG_KEY_IMPORT_DIR=/etc/gpg
+
+usage()
+{
+ echo "usage: publish-java-sdk.sh
+
+ --revision Value for the revision e.g. '0.2.3'
+ --gpg-key-import-dir Directory containing existing GPG keys to import.
+ The directory should contain these 2 files:
+ - public-key
+ - private-key
+ The default value is '/etc/gpg'
+
+ This script assumes the GPG private key is protected by a passphrase.
+ The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml
+ file, credentials to upload releases to Sonatype must also be provided.
+
+ # Example settings: ~/.m2/settings.xml
+
+
+
+ ossrh
+ SONATYPE_USER
+ SONATYPE_PASSWORD
+
+
+
+
+ ossrh
+
+ GPG_PASSPHRASE
+
+
+
+
+"
+}
+
+while [ "$1" != "" ]; do
+ case "$1" in
+ --revision ) REVISION="$2"; shift;;
+ --gpg-key-import-dir ) GPG_KEY_IMPORT_DIR="$2"; shift;;
+ -h | --help ) usage; exit;;
+ * ) usage; exit 1
+ esac
+ shift
+done
+
+if [ -z $REVISION ]; then usage; exit 1; fi
+
+echo "============================================================"
+echo "Checking Maven and GPG versions"
+echo "============================================================"
+mvn --version
+echo ""
+gpg --version
+
+echo "============================================================"
+echo "Importing GPG keys"
+echo "============================================================"
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
+
+echo "============================================================"
+echo "Deploying Java SDK with revision: $REVISION"
+echo "============================================================"
+mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy
diff --git a/.prow/scripts/publish-python-sdk.sh b/infra/scripts/publish-python-sdk.sh
similarity index 100%
rename from .prow/scripts/publish-python-sdk.sh
rename to infra/scripts/publish-python-sdk.sh
diff --git a/.prow/scripts/sync-helm-charts.sh b/infra/scripts/sync-helm-charts.sh
similarity index 96%
rename from .prow/scripts/sync-helm-charts.sh
rename to infra/scripts/sync-helm-charts.sh
index 88fc04f7d14..8c242aeae69 100755
--- a/.prow/scripts/sync-helm-charts.sh
+++ b/infra/scripts/sync-helm-charts.sh
@@ -31,7 +31,7 @@ fi
exit_code=0
for dir in "$repo_dir"/*; do
- if helm dependency build "$dir"; then
+ if helm dep update "$dir" && helm dep build "$dir"; then
helm package --destination "$sync_dir" "$dir"
else
log_error "Problem building dependencies. Skipping packaging of '$dir'."
diff --git a/.prow/scripts/test-core-ingestion.sh b/infra/scripts/test-core-ingestion.sh
similarity index 95%
rename from .prow/scripts/test-core-ingestion.sh
rename to infra/scripts/test-core-ingestion.sh
index 98a47ca68c9..b7ec1d814b8 100755
--- a/.prow/scripts/test-core-ingestion.sh
+++ b/infra/scripts/test-core-ingestion.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
--output-dir /root/
diff --git a/.prow/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh
similarity index 94%
rename from .prow/scripts/test-end-to-end-batch.sh
rename to infra/scripts/test-end-to-end-batch.sh
index b370c5b045b..49293826940 100755
--- a/.prow/scripts/test-end-to-end-batch.sh
+++ b/infra/scripts/test-end-to-end-batch.sh
@@ -3,12 +3,6 @@
set -e
set -o pipefail
-if ! cat /etc/*release | grep -q stretch; then
- echo ${BASH_SOURCE} only supports Debian stretch.
- echo Please change your operating system to use this script.
- exit 1
-fi
-
echo "
This script will run end-to-end tests for Feast Core and Batch Serving.
@@ -85,7 +79,10 @@ Building jars for Feast
============================================================
"
-.prow/scripts/download-maven-cache.sh \
+FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+echo Building Jars for version: $FEAST_BUILD_VERSION
+
+infra/scripts/download-maven-cache.sh \
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
--output-dir /root/
@@ -138,7 +135,7 @@ management:
enabled: false
EOF
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-${FEAST_BUILD_VERSION}.jar \
--spring.config.location=file:///tmp/core.application.yml \
&> /var/log/feast-core.log &
sleep 30
@@ -191,7 +188,7 @@ spring:
web-environment: false
EOF
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-${FEAST_BUILD_VERSION}.jar \
--spring.config.location=file:///tmp/serving.warehouse.application.yml \
&> /var/log/feast-serving-warehouse.log &
sleep 15
diff --git a/.prow/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
similarity index 93%
rename from .prow/scripts/test-end-to-end.sh
rename to infra/scripts/test-end-to-end.sh
index 2c6f4a098f9..8843e2fa13a 100755
--- a/.prow/scripts/test-end-to-end.sh
+++ b/infra/scripts/test-end-to-end.sh
@@ -3,12 +3,6 @@
set -e
set -o pipefail
-if ! cat /etc/*release | grep -q stretch; then
- echo ${BASH_SOURCE} only supports Debian stretch.
- echo Please change your operating system to use this script.
- exit 1
-fi
-
echo "
This script will run end-to-end tests for Feast Core and Online Serving.
@@ -68,13 +62,19 @@ Building jars for Feast
============================================================
"
-.prow/scripts/download-maven-cache.sh \
+FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+echo Building Jars for version: $FEAST_BUILD_VERSION
+
+infra/scripts/download-maven-cache.sh \
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
--output-dir /root/
# Build jars for Feast
mvn --quiet --batch-mode --define skipTests=true clean package
+ls -lh core/target/*jar
+ls -lh serving/target/*jar
+
echo "
============================================================
Starting Feast Core
@@ -121,7 +121,7 @@ management:
enabled: false
EOF
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-$FEAST_BUILD_VERSION.jar \
--spring.config.location=file:///tmp/core.application.yml \
&> /var/log/feast-core.log &
sleep 30
@@ -172,7 +172,7 @@ spring:
web-environment: false
EOF
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-$FEAST_BUILD_VERSION.jar \
--spring.config.location=file:///tmp/serving.online.application.yml \
&> /var/log/feast-serving-online.log &
sleep 15
diff --git a/.prow/scripts/test-golang-sdk.sh b/infra/scripts/test-golang-sdk.sh
similarity index 100%
rename from .prow/scripts/test-golang-sdk.sh
rename to infra/scripts/test-golang-sdk.sh
diff --git a/.prow/scripts/test-java-sdk.sh b/infra/scripts/test-java-sdk.sh
similarity index 100%
rename from .prow/scripts/test-java-sdk.sh
rename to infra/scripts/test-java-sdk.sh
diff --git a/.prow/scripts/test-python-sdk.sh b/infra/scripts/test-python-sdk.sh
similarity index 100%
rename from .prow/scripts/test-python-sdk.sh
rename to infra/scripts/test-python-sdk.sh
diff --git a/.prow/scripts/test-serving.sh b/infra/scripts/test-serving.sh
similarity index 90%
rename from .prow/scripts/test-serving.sh
rename to infra/scripts/test-serving.sh
index b56001619b3..ce9dc0a8162 100755
--- a/.prow/scripts/test-serving.sh
+++ b/infra/scripts/test-serving.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
--output-dir /root/
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index eb892335180..b6f1ea371a4 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -21,7 +21,7 @@
4.0.0
- feast
+ dev.feast
feast-parent
${revision}
@@ -31,10 +31,6 @@
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
-
org.apache.maven.plugins
maven-shade-plugin
@@ -91,21 +87,9 @@
- org.glassfish
- javax.el
- 3.0.0
-
-
-
- javax.validation
- validation-api
- 2.0.1.Final
-
-
-
- org.hibernate.validator
- hibernate-validator
- 6.0.13.Final
+ dev.feast
+ datatypes-java
+ ${project.version}
@@ -120,15 +104,6 @@
provided
-
- io.grpc
- grpc-stub
-
-
-
- com.google.cloud
- google-cloud-storage
-
com.google.cloud
google-cloud-bigquery
@@ -148,27 +123,6 @@
mockito-core
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
- com.fasterxml.jackson.dataformat
- jackson-dataformat-yaml
-
-
- com.fasterxml.jackson.module
- jackson-module-jsonSchema
-
-
com.google.protobuf
protobuf-java
@@ -213,6 +167,12 @@
${org.apache.beam.version}
+
+ org.apache.beam
+ beam-sdks-java-io-cassandra
+ ${org.apache.beam.version}
+
+
redis.clients
jedis
@@ -245,6 +205,13 @@
test
+
+ org.cassandraunit
+ cassandra-unit-shaded
+ 3.11.2.0
+ test
+
+
com.google.guava
guava
diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java
new file mode 100644
index 00000000000..b1e5c4f0ce5
--- /dev/null
+++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.transform;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.mapping.Mapper.Option;
+import feast.store.serving.cassandra.CassandraMutation;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.concurrent.Future;
+import org.apache.beam.sdk.io.cassandra.Mapper;
+
+/** A {@link Mapper} that supports writing {@code CassandraMutation}s with the Beam Cassandra IO. */
+public class CassandraMutationMapper implements Mapper, Serializable {
+
+ private com.datastax.driver.mapping.Mapper mapper;
+
+ CassandraMutationMapper(com.datastax.driver.mapping.Mapper mapper) {
+ this.mapper = mapper;
+ }
+
+ @Override
+ public Iterator map(ResultSet resultSet) {
+ throw new UnsupportedOperationException("Only supports write operations");
+ }
+
+ @Override
+ public Future deleteAsync(CassandraMutation entityClass) {
+ throw new UnsupportedOperationException("Only supports write operations");
+ }
+
+ /**
+ * Saves records to Cassandra with: - Cassandra's internal write time set to the timestamp of the
+ * record. Cassandra will not override an existing record with the same partition key if the write
+ * time is older - Expiration of the record
+ *
+ * @param entityClass Cassandra's object mapper
+ */
+ @Override
+ public Future saveAsync(CassandraMutation entityClass) {
+ return mapper.saveAsync(
+ entityClass,
+ Option.timestamp(entityClass.getWriteTime()),
+ Option.ttl(entityClass.getTtl()));
+ }
+}
diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java
new file mode 100644
index 00000000000..9f344650995
--- /dev/null
+++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.transform;
+
+import com.datastax.driver.core.Session;
+import com.datastax.driver.mapping.MappingManager;
+import feast.store.serving.cassandra.CassandraMutation;
+import org.apache.beam.sdk.io.cassandra.Mapper;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+
+public class CassandraMutationMapperFactory implements SerializableFunction {
+
+ private transient MappingManager mappingManager;
+ private Class entityClass;
+
+ public CassandraMutationMapperFactory(Class entityClass) {
+ this.entityClass = entityClass;
+ }
+
+ @Override
+ public Mapper apply(Session session) {
+ if (mappingManager == null) {
+ this.mappingManager = new MappingManager(session);
+ }
+
+ return new CassandraMutationMapper(mappingManager.mapper(entityClass));
+ }
+}
diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java
index 6f697f1c6fd..c89b95bfb22 100644
--- a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java
+++ b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java
@@ -16,12 +16,14 @@
*/
package feast.ingestion.transform;
+import com.datastax.driver.core.Session;
import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors;
import com.google.api.services.bigquery.model.TableRow;
import com.google.auto.value.AutoValue;
import feast.core.FeatureSetProto.FeatureSetSpec;
import feast.core.StoreProto.Store;
import feast.core.StoreProto.Store.BigQueryConfig;
+import feast.core.StoreProto.Store.CassandraConfig;
import feast.core.StoreProto.Store.RedisConfig;
import feast.core.StoreProto.Store.StoreType;
import feast.ingestion.options.ImportOptions;
@@ -29,11 +31,16 @@
import feast.ingestion.values.FailedElement;
import feast.store.serving.bigquery.FeatureRowToTableRow;
import feast.store.serving.bigquery.GetTableDestination;
+import feast.store.serving.cassandra.CassandraMutation;
+import feast.store.serving.cassandra.FeatureRowToCassandraMutationDoFn;
import feast.store.serving.redis.FeatureRowToRedisMutationDoFn;
import feast.store.serving.redis.RedisCustomIO;
import feast.types.FeatureRowProto.FeatureRow;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Map;
+import org.apache.beam.sdk.io.cassandra.CassandraIO;
+import org.apache.beam.sdk.io.cassandra.Mapper;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method;
@@ -47,10 +54,10 @@
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.TypeDescriptors;
-import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.slf4j.Logger;
@AutoValue
@@ -61,8 +68,8 @@ public abstract class WriteToStore extends PTransform, P
public static final String METRIC_NAMESPACE = "WriteToStore";
public static final String ELEMENTS_WRITTEN_METRIC = "elements_written";
- private static final Counter elementsWritten = Metrics
- .counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC);
+ private static final Counter elementsWritten =
+ Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC);
public abstract Store getStore();
@@ -146,16 +153,37 @@ public void processElement(ProcessContext context) {
.build());
}
break;
+ case CASSANDRA:
+ CassandraConfig cassandraConfig = getStore().getCassandraConfig();
+ SerializableFunction mapperFactory =
+ new CassandraMutationMapperFactory(CassandraMutation.class);
+ input
+ .apply(
+ "Create CassandraMutation from FeatureRow",
+ ParDo.of(
+ new FeatureRowToCassandraMutationDoFn(
+ getFeatureSetSpecs(), cassandraConfig.getDefaultTtl())))
+ .apply(
+ CassandraIO.write()
+ .withHosts(Arrays.asList(cassandraConfig.getBootstrapHosts().split(",")))
+ .withPort(cassandraConfig.getPort())
+ .withKeyspace(cassandraConfig.getKeyspace())
+ .withEntity(CassandraMutation.class)
+ .withMapperFactoryFn(mapperFactory));
+ break;
default:
log.error("Store type '{}' is not supported. No Feature Row will be written.", storeType);
break;
}
- input.apply("IncrementWriteToStoreElementsWrittenCounter",
- MapElements.into(TypeDescriptors.booleans()).via((FeatureRow row) -> {
- elementsWritten.inc();
- return true;
- }));
+ input.apply(
+ "IncrementWriteToStoreElementsWrittenCounter",
+ MapElements.into(TypeDescriptors.booleans())
+ .via(
+ (FeatureRow row) -> {
+ elementsWritten.inc();
+ return true;
+ }));
return PDone.in(input.getPipeline());
}
diff --git a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java
index 5ceb8bd2f96..3478227165b 100644
--- a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java
+++ b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java
@@ -18,6 +18,14 @@
import static feast.types.ValueProto.ValueType;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.KeyspaceMetadata;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.schemabuilder.Create;
+import com.datastax.driver.core.schemabuilder.KeyspaceOptions;
+import com.datastax.driver.core.schemabuilder.SchemaBuilder;
+import com.datastax.driver.mapping.MappingManager;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.DatasetId;
@@ -39,13 +47,19 @@
import feast.core.FeatureSetProto.FeatureSetSpec;
import feast.core.FeatureSetProto.FeatureSpec;
import feast.core.StoreProto.Store;
+import feast.core.StoreProto.Store.CassandraConfig;
import feast.core.StoreProto.Store.RedisConfig;
import feast.core.StoreProto.Store.StoreType;
+import feast.store.serving.cassandra.CassandraMutation;
import feast.types.ValueProto.ValueType.Enum;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import redis.clients.jedis.JedisPool;
@@ -112,6 +126,9 @@ public static void setupStore(Store store, FeatureSetSpec featureSetSpec) {
store.getBigqueryConfig().getDatasetId(),
BigQueryOptions.getDefaultInstance().getService());
break;
+ case CASSANDRA:
+ StoreUtil.setupCassandra(store.getCassandraConfig());
+ break;
default:
log.warn("Store type '{}' is unsupported", storeType);
break;
@@ -246,4 +263,74 @@ public static void checkRedisConnection(RedisConfig redisConfig) {
}
jedisPool.close();
}
+
+ /**
+ * Ensures Cassandra is accessible, else throw a RuntimeException. Creates Cassandra keyspace and
+ * table if it does not already exist
+ *
+ * @param cassandraConfig Please refer to feast.core.Store proto
+ */
+ public static void setupCassandra(CassandraConfig cassandraConfig) {
+ List contactPoints =
+ Arrays.stream(cassandraConfig.getBootstrapHosts().split(","))
+ .map(host -> new InetSocketAddress(host, cassandraConfig.getPort()))
+ .collect(Collectors.toList());
+ Cluster cluster = Cluster.builder().addContactPointsWithPorts(contactPoints).build();
+ Session session;
+
+ try {
+ String keyspace = cassandraConfig.getKeyspace();
+ KeyspaceMetadata keyspaceMetadata = cluster.getMetadata().getKeyspace(keyspace);
+ if (keyspaceMetadata == null) {
+ log.info("Creating keyspace '{}'", keyspace);
+ Map replicationOptions =
+ cassandraConfig.getReplicationOptionsMap().entrySet().stream()
+ .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
+ KeyspaceOptions createKeyspace =
+ SchemaBuilder.createKeyspace(keyspace)
+ .ifNotExists()
+ .with()
+ .replication(replicationOptions);
+ session = cluster.newSession();
+ session.execute(createKeyspace);
+ }
+
+ session = cluster.connect(keyspace);
+ // Currently no support for creating table from entity mapper:
+ // https://datastax-oss.atlassian.net/browse/JAVA-569
+ Create createTable =
+ SchemaBuilder.createTable(keyspace, cassandraConfig.getTableName())
+ .ifNotExists()
+ .addPartitionKey(CassandraMutation.ENTITIES, DataType.text())
+ .addClusteringColumn(CassandraMutation.FEATURE, DataType.text())
+ .addColumn(CassandraMutation.VALUE, DataType.blob());
+ log.info("Create Cassandra table if not exists..");
+ session.execute(createTable);
+
+ validateCassandraTable(session);
+
+ session.close();
+ } catch (RuntimeException e) {
+ throw new RuntimeException(
+ String.format(
+ "Failed to connect to Cassandra at bootstrap hosts: '%s' port: '%s'. Please check that your Cassandra is running and accessible from Feast.",
+ contactPoints.stream()
+ .map(InetSocketAddress::getHostName)
+ .collect(Collectors.joining(",")),
+ cassandraConfig.getPort()),
+ e);
+ }
+ cluster.close();
+ }
+
+ private static void validateCassandraTable(Session session) {
+ try {
+ new MappingManager(session).mapper(CassandraMutation.class).getTableMetadata();
+ } catch (RuntimeException e) {
+ throw new RuntimeException(
+ String.format(
+ "Table created does not match the datastax object mapper: %s",
+ CassandraMutation.class.getSimpleName()));
+ }
+ }
}
diff --git a/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java
new file mode 100644
index 00000000000..87a327e7726
--- /dev/null
+++ b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.utils;
+
+import feast.types.ValueProto.Value;
+
+/**
+ * Utility class for converting {@link Value} of different types to a string for storing as key in
+ * data stores
+ */
+public class ValueUtil {
+
+ public static String toString(Value value) {
+ String strValue;
+ switch (value.getValCase()) {
+ case BYTES_VAL:
+ strValue = value.getBytesVal().toString();
+ break;
+ case STRING_VAL:
+ strValue = value.getStringVal();
+ break;
+ case INT32_VAL:
+ strValue = String.valueOf(value.getInt32Val());
+ break;
+ case INT64_VAL:
+ strValue = String.valueOf(value.getInt64Val());
+ break;
+ case DOUBLE_VAL:
+ strValue = String.valueOf(value.getDoubleVal());
+ break;
+ case FLOAT_VAL:
+ strValue = String.valueOf(value.getFloatVal());
+ break;
+ case BOOL_VAL:
+ strValue = String.valueOf(value.getBoolVal());
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format("toString method not supported for type %s", value.getValCase()));
+ }
+ return strValue;
+ }
+}
diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java
new file mode 100644
index 00000000000..23bbb9c3b31
--- /dev/null
+++ b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java
@@ -0,0 +1,121 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.store.serving.cassandra;
+
+import com.datastax.driver.mapping.annotations.ClusteringColumn;
+import com.datastax.driver.mapping.annotations.Computed;
+import com.datastax.driver.mapping.annotations.PartitionKey;
+import com.datastax.driver.mapping.annotations.Table;
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.ingestion.utils.ValueUtil;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.FieldProto.Field;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.DefaultCoder;
+
+/**
+ * Cassandra's object mapper that handles basic CRUD operations in Cassandra tables More info:
+ * https://docs.datastax.com/en/developer/java-driver/3.1/manual/object_mapper/
+ */
+@DefaultCoder(value = AvroCoder.class)
+@Table(name = "feature_store")
+public final class CassandraMutation implements Serializable {
+
+ public static final String ENTITIES = "entities";
+ public static final String FEATURE = "feature";
+ public static final String VALUE = "value";
+
+ @PartitionKey private final String entities;
+
+ @ClusteringColumn private final String feature;
+
+ private final ByteBuffer value;
+
+ @Computed(value = "writetime(value)")
+ private final long writeTime;
+
+ @Computed(value = "ttl(value)")
+ private final int ttl;
+
+ // NoArgs constructor is needed when using Beam's CassandraIO withEntity and specifying this
+ // class,
+ // it looks for an init() method
+ CassandraMutation() {
+ this.entities = null;
+ this.feature = null;
+ this.value = null;
+ this.writeTime = 0;
+ this.ttl = 0;
+ }
+
+ CassandraMutation(String entities, String feature, ByteBuffer value, long writeTime, int ttl) {
+ this.entities = entities;
+ this.feature = feature;
+ this.value = value;
+ this.writeTime = writeTime;
+ this.ttl = ttl;
+ }
+
+ public long getWriteTime() {
+ return writeTime;
+ }
+
+ public int getTtl() {
+ return ttl;
+ }
+
+ static String keyFromFeatureRow(FeatureSetSpec featureSetSpec, FeatureRow featureRow) {
+ Set entityNames =
+ featureSetSpec.getEntitiesList().stream()
+ .map(EntitySpec::getName)
+ .collect(Collectors.toSet());
+ List entities = new ArrayList<>();
+ for (Field field : featureRow.getFieldsList()) {
+ if (entityNames.contains(field.getName())) {
+ entities.add(field);
+ }
+ }
+ return featureRow.getFeatureSet()
+ + ":"
+ + entities.stream()
+ .map(f -> f.getName() + "=" + ValueUtil.toString(f.getValue()))
+ .collect(Collectors.joining("|"));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o instanceof CassandraMutation) {
+ CassandraMutation that = (CassandraMutation) o;
+ return this.entities.equals(that.entities)
+ && this.feature.equals(that.feature)
+ && this.value.equals(that.value)
+ && this.writeTime == that.writeTime
+ && this.ttl == that.ttl;
+ }
+ return false;
+ }
+}
diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java
new file mode 100644
index 00000000000..177a8703e68
--- /dev/null
+++ b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java
@@ -0,0 +1,85 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.store.serving.cassandra;
+
+import com.google.protobuf.Duration;
+import com.google.protobuf.util.Timestamps;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.FieldProto.Field;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.slf4j.Logger;
+
+public class FeatureRowToCassandraMutationDoFn extends DoFn {
+
+ private static final Logger log =
+ org.slf4j.LoggerFactory.getLogger(FeatureRowToCassandraMutationDoFn.class);
+ private Map featureSetSpecs;
+ private Map maxAges;
+
+ public FeatureRowToCassandraMutationDoFn(Map specs, Duration defaultTtl) {
+ this.featureSetSpecs = specs;
+ this.maxAges = new HashMap<>();
+ for (FeatureSetSpec spec : specs.values()) {
+ String featureSetName = spec.getName() + ":" + spec.getVersion();
+ if (spec.getMaxAge() != null && spec.getMaxAge().getSeconds() > 0) {
+ maxAges.put(featureSetName, Math.toIntExact(spec.getMaxAge().getSeconds()));
+ } else {
+ maxAges.put(featureSetName, Math.toIntExact(defaultTtl.getSeconds()));
+ }
+ }
+ }
+
+ /** Output a Cassandra mutation object for every feature in the feature row. */
+ @ProcessElement
+ public void processElement(ProcessContext context) {
+ FeatureRow featureRow = context.element();
+ try {
+ FeatureSetSpec featureSetSpec = featureSetSpecs.get(featureRow.getFeatureSet());
+ Set featureNames =
+ featureSetSpec.getFeaturesList().stream()
+ .map(FeatureSpec::getName)
+ .collect(Collectors.toSet());
+ String key = CassandraMutation.keyFromFeatureRow(featureSetSpec, featureRow);
+
+ Collection mutations = new ArrayList<>();
+ for (Field field : featureRow.getFieldsList()) {
+ if (featureNames.contains(field.getName())) {
+ mutations.add(
+ new CassandraMutation(
+ key,
+ field.getName(),
+ ByteBuffer.wrap(field.getValue().toByteArray()),
+ Timestamps.toMicros(featureRow.getEventTimestamp()),
+ maxAges.get(featureRow.getFeatureSet())));
+ }
+ }
+
+ mutations.forEach(context::output);
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
index 9bc503f9870..a7471443617 100644
--- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
+++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
@@ -24,8 +24,9 @@
import feast.store.serving.redis.RedisCustomIO.RedisMutation;
import feast.types.FeatureRowProto.FeatureRow;
import feast.types.FieldProto.Field;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.stream.Collectors;
import org.apache.beam.sdk.transforms.DoFn;
import org.slf4j.Logger;
@@ -42,17 +43,24 @@ public FeatureRowToRedisMutationDoFn(Map featureSetSpecs
private RedisKey getKey(FeatureRow featureRow) {
FeatureSetSpec featureSetSpec = featureSetSpecs.get(featureRow.getFeatureSet());
- Set entityNames =
+ List entityNames =
featureSetSpec.getEntitiesList().stream()
.map(EntitySpec::getName)
- .collect(Collectors.toSet());
+ .sorted()
+ .collect(Collectors.toList());
+ Map entityFields = new HashMap<>();
Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet());
for (Field field : featureRow.getFieldsList()) {
if (entityNames.contains(field.getName())) {
- redisKeyBuilder.addEntities(field);
+ entityFields.putIfAbsent(
+ field.getName(),
+ Field.newBuilder().setName(field.getName()).setValue(field.getValue()).build());
}
}
+ for (String entityName : entityNames) {
+ redisKeyBuilder.addEntities(entityFields.get(entityName));
+ }
return redisKeyBuilder.build();
}
diff --git a/ingestion/src/main/proto/feast b/ingestion/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/ingestion/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
deleted file mode 100644
index cb64dd715f6..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-import "google/protobuf/timestamp.proto";
-import "feast/types/Field.proto";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceAccumProto";
-
-// Accumlator for merging feature rows.
-message CoalesceAccum {
- string entityKey = 1;
- google.protobuf.Timestamp eventTimestamp = 3;
- string entityName = 4;
-
- map features = 6;
- // map of features to their counter values when they were last added to accumulator
- map featureMarks = 7;
- int64 counter = 8;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
deleted file mode 100644
index 9730b49ec3b..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceKeyProto";
-
-message CoalesceKey {
- string entityName = 1;
- string entityKey = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/third_party b/ingestion/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/ingestion/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
index bd034341ec9..4a09bee82ff 100644
--- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
+++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
@@ -170,12 +170,14 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
Map expected = new HashMap<>();
LOGGER.info("Generating test data ...");
- IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE).forEach(i -> {
- FeatureRow randomRow = TestUtil.createRandomFeatureRow(spec);
- RedisKey redisKey = TestUtil.createRedisKey(spec, randomRow);
- input.add(randomRow);
- expected.put(redisKey, randomRow);
- });
+ IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE)
+ .forEach(
+ i -> {
+ FeatureRow randomRow = TestUtil.createRandomFeatureRow(spec);
+ RedisKey redisKey = TestUtil.createRedisKey(spec, randomRow);
+ input.add(randomRow);
+ expected.put(redisKey, randomRow);
+ });
LOGGER.info("Starting Import Job with the following options: {}", options.toString());
PipelineResult pipelineResult = ImportJob.runPipeline(options);
@@ -183,43 +185,50 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
Assert.assertEquals(pipelineResult.getState(), State.RUNNING);
LOGGER.info("Publishing {} Feature Row messages to Kafka ...", input.size());
- TestUtil.publishFeatureRowsToKafka(KAFKA_BOOTSTRAP_SERVERS, KAFKA_TOPIC, input,
- ByteArraySerializer.class, KAFKA_PUBLISH_TIMEOUT_SEC);
- TestUtil.waitUntilAllElementsAreWrittenToStore(pipelineResult,
+ TestUtil.publishFeatureRowsToKafka(
+ KAFKA_BOOTSTRAP_SERVERS,
+ KAFKA_TOPIC,
+ input,
+ ByteArraySerializer.class,
+ KAFKA_PUBLISH_TIMEOUT_SEC);
+ TestUtil.waitUntilAllElementsAreWrittenToStore(
+ pipelineResult,
Duration.standardSeconds(IMPORT_JOB_MAX_RUN_DURATION_SEC),
Duration.standardSeconds(IMPORT_JOB_CHECK_INTERVAL_DURATION_SEC));
LOGGER.info("Validating the actual values written to Redis ...");
Jedis jedis = new Jedis(REDIS_HOST, REDIS_PORT);
- expected.forEach((key, expectedValue) -> {
-
- // Ensure ingested key exists.
- byte[] actualByteValue = jedis.get(key.toByteArray());
- if (actualByteValue == null) {
- LOGGER.error("Key not found in Redis: " + key);
- LOGGER.info("Redis INFO:");
- LOGGER.info(jedis.info());
- String randomKey = jedis.randomKey();
- if (randomKey != null) {
- LOGGER.info("Sample random key, value (for debugging purpose):");
- LOGGER.info("Key: " + randomKey);
- LOGGER.info("Value: " + jedis.get(randomKey));
- }
- Assert.fail("Missing key in Redis.");
- }
-
- // Ensure value is a valid serialized FeatureRow object.
- FeatureRow actualValue = null;
- try {
- actualValue = FeatureRow.parseFrom(actualByteValue);
- } catch (InvalidProtocolBufferException e) {
- Assert.fail(String
- .format("Actual Redis value cannot be parsed as FeatureRow, key: %s, value :%s",
- key, new String(actualByteValue, StandardCharsets.UTF_8)));
- }
-
- // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow.
- Assert.assertEquals(expectedValue, actualValue);
- });
+ expected.forEach(
+ (key, expectedValue) -> {
+
+ // Ensure ingested key exists.
+ byte[] actualByteValue = jedis.get(key.toByteArray());
+ if (actualByteValue == null) {
+ LOGGER.error("Key not found in Redis: " + key);
+ LOGGER.info("Redis INFO:");
+ LOGGER.info(jedis.info());
+ String randomKey = jedis.randomKey();
+ if (randomKey != null) {
+ LOGGER.info("Sample random key, value (for debugging purpose):");
+ LOGGER.info("Key: " + randomKey);
+ LOGGER.info("Value: " + jedis.get(randomKey));
+ }
+ Assert.fail("Missing key in Redis.");
+ }
+
+ // Ensure value is a valid serialized FeatureRow object.
+ FeatureRow actualValue = null;
+ try {
+ actualValue = FeatureRow.parseFrom(actualByteValue);
+ } catch (InvalidProtocolBufferException e) {
+ Assert.fail(
+ String.format(
+ "Actual Redis value cannot be parsed as FeatureRow, key: %s, value :%s",
+ key, new String(actualByteValue, StandardCharsets.UTF_8)));
+ }
+
+ // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow.
+ Assert.assertEquals(expectedValue, actualValue);
+ });
}
}
diff --git a/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java
new file mode 100644
index 00000000000..da914ca1757
--- /dev/null
+++ b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java
@@ -0,0 +1,248 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.transform;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.google.protobuf.InvalidProtocolBufferException;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.StoreProto.Store;
+import feast.core.StoreProto.Store.CassandraConfig;
+import feast.core.StoreProto.Store.StoreType;
+import feast.test.TestUtil;
+import feast.test.TestUtil.LocalCassandra;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.FieldProto.Field;
+import feast.types.ValueProto.Value;
+import feast.types.ValueProto.ValueType.Enum;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class CassandraWriteToStoreIT implements Serializable {
+ private FeatureSetSpec featureSetSpec;
+ private FeatureRow row;
+
+ class FakeCassandraWriteToStore extends WriteToStore {
+
+ private FeatureSetSpec featureSetSpec;
+
+ FakeCassandraWriteToStore(FeatureSetSpec featureSetSpec) {
+ this.featureSetSpec = featureSetSpec;
+ }
+
+ @Override
+ public Store getStore() {
+ return Store.newBuilder()
+ .setType(StoreType.CASSANDRA)
+ .setName("SERVING")
+ .setCassandraConfig(getCassandraConfig())
+ .build();
+ }
+
+ @Override
+ public Map getFeatureSetSpecs() {
+ return new HashMap() {
+ {
+ put(featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), featureSetSpec);
+ }
+ };
+ }
+ }
+
+ private static CassandraConfig getCassandraConfig() {
+ return CassandraConfig.newBuilder()
+ .setBootstrapHosts(LocalCassandra.getHost())
+ .setPort(LocalCassandra.getPort())
+ .setTableName("feature_store")
+ .setKeyspace("test")
+ .putAllReplicationOptions(
+ new HashMap() {
+ {
+ put("class", "SimpleStrategy");
+ put("replication_factor", "1");
+ }
+ })
+ .build();
+ }
+
+ @BeforeClass
+ public static void startServer() throws InterruptedException, IOException, TTransportException {
+ LocalCassandra.start();
+ LocalCassandra.createKeyspaceAndTable(getCassandraConfig());
+ }
+
+ @Before
+ public void setUp() {
+ featureSetSpec =
+ TestUtil.createFeatureSetSpec(
+ "fs",
+ 1,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", Enum.INT64);
+ put("entity2", Enum.STRING);
+ }
+ },
+ new HashMap() {
+ {
+ put("feature1", Enum.INT64);
+ put("feature2", Enum.INT64);
+ }
+ });
+ row =
+ TestUtil.createFeatureRow(
+ featureSetSpec,
+ 100,
+ new HashMap() {
+ {
+ put("entity1", TestUtil.intValue(1));
+ put("entity2", TestUtil.strValue("a"));
+ put("feature1", TestUtil.intValue(1));
+ put("feature2", TestUtil.intValue(2));
+ }
+ });
+ }
+
+ @Rule public transient TestPipeline testPipeline = TestPipeline.create();
+
+ @AfterClass
+ public static void cleanUp() {
+ LocalCassandra.stop();
+ }
+
+ @Test
+ public void testWriteCassandra_happyPath() throws InvalidProtocolBufferException {
+ PCollection input = testPipeline.apply(Create.of(row));
+
+ input.apply(new FakeCassandraWriteToStore(featureSetSpec));
+
+ testPipeline.run();
+
+ ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store");
+ List actualResults = getResults(resultSet);
+
+ List expectedFields =
+ Arrays.asList(
+ Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(),
+ Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build());
+
+ assertTrue(actualResults.containsAll(expectedFields));
+ assertEquals(expectedFields.size(), actualResults.size());
+ }
+
+ @Test(timeout = 30000)
+ public void testWriteCassandra_shouldNotRetrieveExpiredValues()
+ throws InvalidProtocolBufferException {
+ // Set max age to 1 second
+ FeatureSetSpec featureSetSpec =
+ TestUtil.createFeatureSetSpec(
+ "fs",
+ 1,
+ 1,
+ new HashMap() {
+ {
+ put("entity1", Enum.INT64);
+ put("entity2", Enum.STRING);
+ }
+ },
+ new HashMap() {
+ {
+ put("feature1", Enum.INT64);
+ put("feature2", Enum.INT64);
+ }
+ });
+
+ PCollection input = testPipeline.apply(Create.of(row));
+
+ input.apply(new FakeCassandraWriteToStore(featureSetSpec));
+
+ testPipeline.run();
+
+ while (true) {
+ ResultSet resultSet =
+ LocalCassandra.getSession()
+ .execute("SELECT feature, value, ttl(value) as expiry FROM test.feature_store");
+ List results = getResults(resultSet);
+ if (results.isEmpty()) break;
+ }
+ }
+
+ @Test
+ public void testWriteCassandra_shouldNotOverrideNewerValues()
+ throws InvalidProtocolBufferException {
+ FeatureRow olderRow =
+ TestUtil.createFeatureRow(
+ featureSetSpec,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", TestUtil.intValue(1));
+ put("entity2", TestUtil.strValue("a"));
+ put("feature1", TestUtil.intValue(3));
+ put("feature2", TestUtil.intValue(4));
+ }
+ });
+
+ PCollection input = testPipeline.apply(Create.of(row, olderRow));
+
+ input.apply(new FakeCassandraWriteToStore(featureSetSpec));
+
+ testPipeline.run();
+
+ ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store");
+ List actualResults = getResults(resultSet);
+
+ List expectedFields =
+ Arrays.asList(
+ Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(),
+ Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build());
+
+ assertTrue(actualResults.containsAll(expectedFields));
+ assertEquals(expectedFields.size(), actualResults.size());
+ }
+
+ private List getResults(ResultSet resultSet) throws InvalidProtocolBufferException {
+ List results = new ArrayList<>();
+ while (!resultSet.isExhausted()) {
+ Row row = resultSet.one();
+ results.add(
+ Field.newBuilder()
+ .setName(row.getString("feature"))
+ .setValue(Value.parseFrom(row.getBytes("value")))
+ .build());
+ }
+ return results;
+ }
+}
diff --git a/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java b/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java
new file mode 100644
index 00000000000..8c6874ecac7
--- /dev/null
+++ b/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java
@@ -0,0 +1,167 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.util;
+
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.TableMetadata;
+import com.datastax.driver.core.schemabuilder.Create;
+import com.datastax.driver.core.schemabuilder.SchemaBuilder;
+import feast.core.StoreProto.Store.CassandraConfig;
+import feast.ingestion.utils.StoreUtil;
+import feast.store.serving.cassandra.CassandraMutation;
+import feast.test.TestUtil.LocalCassandra;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CassandraStoreUtilIT {
+
+ @BeforeClass
+ public static void startServer() throws InterruptedException, IOException, TTransportException {
+ LocalCassandra.start();
+ }
+
+ @After
+ public void teardown() {
+ LocalCassandra.stop();
+ }
+
+ @Test
+ public void setupCassandra_shouldCreateKeyspaceAndTable() {
+ CassandraConfig config =
+ CassandraConfig.newBuilder()
+ .setBootstrapHosts(LocalCassandra.getHost())
+ .setPort(LocalCassandra.getPort())
+ .setKeyspace("test")
+ .setTableName("feature_store")
+ .putAllReplicationOptions(
+ new HashMap() {
+ {
+ put("class", "NetworkTopologyStrategy");
+ put("dc1", "2");
+ put("dc2", "3");
+ }
+ })
+ .build();
+ StoreUtil.setupCassandra(config);
+
+ Map actualReplication =
+ LocalCassandra.getCluster().getMetadata().getKeyspace("test").getReplication();
+ Map expectedReplication =
+ new HashMap() {
+ {
+ put("class", "org.apache.cassandra.locator.NetworkTopologyStrategy");
+ put("dc1", "2");
+ put("dc2", "3");
+ }
+ };
+ TableMetadata tableMetadata =
+ LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store");
+
+ Assert.assertEquals(expectedReplication, actualReplication);
+ Assert.assertNotNull(tableMetadata);
+ }
+
+ @Test
+ public void setupCassandra_shouldBeIdempotent_whenTableAlreadyExistsAndSchemaMatches() {
+ CassandraConfig config =
+ CassandraConfig.newBuilder()
+ .setBootstrapHosts(LocalCassandra.getHost())
+ .setPort(LocalCassandra.getPort())
+ .setKeyspace("test")
+ .setTableName("feature_store")
+ .putAllReplicationOptions(
+ new HashMap() {
+ {
+ put("class", "SimpleStrategy");
+ put("replication_factor", "2");
+ }
+ })
+ .build();
+
+ LocalCassandra.createKeyspaceAndTable(config);
+
+ // Check table is created
+ Assert.assertNotNull(
+ LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store"));
+
+ StoreUtil.setupCassandra(config);
+
+ Assert.assertNotNull(
+ LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store"));
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void setupCassandra_shouldThrowException_whenTableNameDoesNotMatchObjectMapper() {
+ CassandraConfig config =
+ CassandraConfig.newBuilder()
+ .setBootstrapHosts(LocalCassandra.getHost())
+ .setPort(LocalCassandra.getPort())
+ .setKeyspace("test")
+ .setTableName("test_data_store")
+ .putAllReplicationOptions(
+ new HashMap() {
+ {
+ put("class", "NetworkTopologyStrategy");
+ put("dc1", "2");
+ put("dc2", "3");
+ }
+ })
+ .build();
+ StoreUtil.setupCassandra(config);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void setupCassandra_shouldThrowException_whenTableSchemaDoesNotMatchObjectMapper() {
+ LocalCassandra.getSession()
+ .execute(
+ "CREATE KEYSPACE test "
+ + "WITH REPLICATION = {"
+ + "'class': 'SimpleStrategy', 'replication_factor': 2 }");
+
+ Create createTable =
+ SchemaBuilder.createTable("test", "feature_store")
+ .ifNotExists()
+ .addPartitionKey(CassandraMutation.ENTITIES, DataType.text())
+ .addClusteringColumn(
+ "featureName", DataType.text()) // Column name does not match in CassandraMutation
+ .addColumn(CassandraMutation.VALUE, DataType.blob());
+ LocalCassandra.getSession().execute(createTable);
+
+ CassandraConfig config =
+ CassandraConfig.newBuilder()
+ .setBootstrapHosts(LocalCassandra.getHost())
+ .setPort(LocalCassandra.getPort())
+ .setKeyspace("test")
+ .setTableName("feature_store")
+ .putAllReplicationOptions(
+ new HashMap() {
+ {
+ put("class", "SimpleStrategy");
+ put("replication_factor", "2");
+ }
+ })
+ .build();
+
+ StoreUtil.setupCassandra(config);
+ }
+}
diff --git a/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java
new file mode 100644
index 00000000000..412c6488f20
--- /dev/null
+++ b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java
@@ -0,0 +1,225 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.store.serving.cassandra;
+
+import com.google.protobuf.Duration;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.test.TestUtil;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.ValueProto.Value;
+import feast.types.ValueProto.ValueType.Enum;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class FeatureRowToCassandraMutationDoFnTest implements Serializable {
+
+ @Rule public transient TestPipeline testPipeline = TestPipeline.create();
+
+ @Test
+ public void processElement_shouldCreateCassandraMutation_givenFeatureRow() {
+ FeatureSetSpec featureSetSpec =
+ TestUtil.createFeatureSetSpec(
+ "fs",
+ 1,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", Enum.INT64);
+ }
+ },
+ new HashMap() {
+ {
+ put("feature1", Enum.STRING);
+ }
+ });
+ FeatureRow featureRow =
+ TestUtil.createFeatureRow(
+ featureSetSpec,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", TestUtil.intValue(1));
+ put("feature1", TestUtil.strValue("a"));
+ }
+ });
+
+ PCollection input = testPipeline.apply(Create.of(featureRow));
+
+ PCollection output =
+ input.apply(
+ ParDo.of(
+ new FeatureRowToCassandraMutationDoFn(
+ new HashMap() {
+ {
+ put(
+ featureSetSpec.getName() + ":" + featureSetSpec.getVersion(),
+ featureSetSpec);
+ }
+ },
+ Duration.newBuilder().setSeconds(0).build())));
+
+ CassandraMutation[] expected =
+ new CassandraMutation[] {
+ new CassandraMutation(
+ "fs:1:entity1=1",
+ "feature1",
+ ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()),
+ 10000000,
+ 10)
+ };
+
+ PAssert.that(output).containsInAnyOrder(expected);
+
+ testPipeline.run();
+ }
+
+ @Test
+ public void
+ processElement_shouldCreateCassandraMutations_givenFeatureRowWithMultipleEntitiesAndFeatures() {
+ FeatureSetSpec featureSetSpec =
+ TestUtil.createFeatureSetSpec(
+ "fs",
+ 1,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", Enum.INT64);
+ put("entity2", Enum.STRING);
+ }
+ },
+ new HashMap() {
+ {
+ put("feature1", Enum.STRING);
+ put("feature2", Enum.INT64);
+ }
+ });
+ FeatureRow featureRow =
+ TestUtil.createFeatureRow(
+ featureSetSpec,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", TestUtil.intValue(1));
+ put("entity2", TestUtil.strValue("b"));
+ put("feature1", TestUtil.strValue("a"));
+ put("feature2", TestUtil.intValue(2));
+ }
+ });
+
+ PCollection input = testPipeline.apply(Create.of(featureRow));
+
+ PCollection output =
+ input.apply(
+ ParDo.of(
+ new FeatureRowToCassandraMutationDoFn(
+ new HashMap() {
+ {
+ put(
+ featureSetSpec.getName() + ":" + featureSetSpec.getVersion(),
+ featureSetSpec);
+ }
+ },
+ Duration.newBuilder().setSeconds(0).build())));
+
+ CassandraMutation[] expected =
+ new CassandraMutation[] {
+ new CassandraMutation(
+ "fs:1:entity1=1|entity2=b",
+ "feature1",
+ ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()),
+ 10000000,
+ 10),
+ new CassandraMutation(
+ "fs:1:entity1=1|entity2=b",
+ "feature2",
+ ByteBuffer.wrap(TestUtil.intValue(2).toByteArray()),
+ 10000000,
+ 10)
+ };
+
+ PAssert.that(output).containsInAnyOrder(expected);
+
+ testPipeline.run();
+ }
+
+ @Test
+ public void processElement_shouldUseDefaultMaxAge_whenMissingMaxAge() {
+ Duration defaultTtl = Duration.newBuilder().setSeconds(500).build();
+ FeatureSetSpec featureSetSpec =
+ TestUtil.createFeatureSetSpec(
+ "fs",
+ 1,
+ 0,
+ new HashMap() {
+ {
+ put("entity1", Enum.INT64);
+ }
+ },
+ new HashMap() {
+ {
+ put("feature1", Enum.STRING);
+ }
+ });
+ FeatureRow featureRow =
+ TestUtil.createFeatureRow(
+ featureSetSpec,
+ 10,
+ new HashMap() {
+ {
+ put("entity1", TestUtil.intValue(1));
+ put("feature1", TestUtil.strValue("a"));
+ }
+ });
+
+ PCollection input = testPipeline.apply(Create.of(featureRow));
+
+ PCollection output =
+ input.apply(
+ ParDo.of(
+ new FeatureRowToCassandraMutationDoFn(
+ new HashMap() {
+ {
+ put(
+ featureSetSpec.getName() + ":" + featureSetSpec.getVersion(),
+ featureSetSpec);
+ }
+ },
+ defaultTtl)));
+
+ CassandraMutation[] expected =
+ new CassandraMutation[] {
+ new CassandraMutation(
+ "fs:1:entity1=1",
+ "feature1",
+ ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()),
+ 10000000,
+ 500)
+ };
+
+ PAssert.that(output).containsInAnyOrder(expected);
+
+ testPipeline.run();
+ }
+}
diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java
new file mode 100644
index 00000000000..f74f7667850
--- /dev/null
+++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java
@@ -0,0 +1,171 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.store.serving.redis;
+
+import static org.junit.Assert.*;
+
+import com.google.protobuf.Timestamp;
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.storage.RedisProto.RedisKey;
+import feast.store.serving.redis.RedisCustomIO.RedisMutation;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.FieldProto.Field;
+import feast.types.ValueProto.Value;
+import feast.types.ValueProto.ValueType.Enum;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.protobuf.ProtoCoder;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class FeatureRowToRedisMutationDoFnTest {
+
+ @Rule public transient TestPipeline p = TestPipeline.create();
+
+ private FeatureSetSpec fs =
+ FeatureSetSpec.newBuilder()
+ .setName("feature_set")
+ .setVersion(1)
+ .addEntities(
+ EntitySpec.newBuilder().setName("entity_id_primary").setValueType(Enum.INT32).build())
+ .addEntities(
+ EntitySpec.newBuilder()
+ .setName("entity_id_secondary")
+ .setValueType(Enum.STRING)
+ .build())
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build())
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build())
+ .build();
+
+ @Test
+ public void shouldConvertRowWithDuplicateEntitiesToValidKey() {
+ Map featureSetSpecs = new HashMap<>();
+ featureSetSpecs.put("feature_set", fs);
+
+ FeatureRow offendingRow =
+ FeatureRow.newBuilder()
+ .setFeatureSet("feature_set")
+ .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
+ .addFields(
+ Field.newBuilder()
+ .setName("entity_id_primary")
+ .setValue(Value.newBuilder().setInt32Val(1)))
+ .addFields(
+ Field.newBuilder()
+ .setName("entity_id_primary")
+ .setValue(Value.newBuilder().setInt32Val(2)))
+ .addFields(
+ Field.newBuilder()
+ .setName("entity_id_secondary")
+ .setValue(Value.newBuilder().setStringVal("a")))
+ .build();
+
+ PCollection output =
+ p.apply(Create.of(Collections.singletonList(offendingRow)))
+ .setCoder(ProtoCoder.of(FeatureRow.class))
+ .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
+
+ RedisKey expectedKey =
+ RedisKey.newBuilder()
+ .setFeatureSet("feature_set")
+ .addEntities(
+ Field.newBuilder()
+ .setName("entity_id_primary")
+ .setValue(Value.newBuilder().setInt32Val(1)))
+ .addEntities(
+ Field.newBuilder()
+ .setName("entity_id_secondary")
+ .setValue(Value.newBuilder().setStringVal("a")))
+ .build();
+
+ PAssert.that(output)
+ .satisfies(
+ (SerializableFunction, Void>)
+ input -> {
+ input.forEach(
+ rm -> {
+ assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
+ assert (Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
+ });
+ return null;
+ });
+ p.run();
+ }
+
+ @Test
+ public void shouldConvertRowWithOutOfOrderEntitiesToValidKey() {
+ Map featureSetSpecs = new HashMap<>();
+ featureSetSpecs.put("feature_set", fs);
+
+ FeatureRow offendingRow =
+ FeatureRow.newBuilder()
+ .setFeatureSet("feature_set")
+ .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
+ .addFields(
+ Field.newBuilder()
+ .setName("entity_id_secondary")
+ .setValue(Value.newBuilder().setStringVal("a")))
+ .addFields(
+ Field.newBuilder()
+ .setName("entity_id_primary")
+ .setValue(Value.newBuilder().setInt32Val(1)))
+ .build();
+
+ PCollection output =
+ p.apply(Create.of(Collections.singletonList(offendingRow)))
+ .setCoder(ProtoCoder.of(FeatureRow.class))
+ .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
+
+ RedisKey expectedKey =
+ RedisKey.newBuilder()
+ .setFeatureSet("feature_set")
+ .addEntities(
+ Field.newBuilder()
+ .setName("entity_id_primary")
+ .setValue(Value.newBuilder().setInt32Val(1)))
+ .addEntities(
+ Field.newBuilder()
+ .setName("entity_id_secondary")
+ .setValue(Value.newBuilder().setStringVal("a")))
+ .build();
+
+ PAssert.that(output)
+ .satisfies(
+ (SerializableFunction, Void>)
+ input -> {
+ input.forEach(
+ rm -> {
+ assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
+ assert (Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
+ });
+ return null;
+ });
+ p.run();
+ }
+}
diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java
index ef41f3950a5..51d47127599 100644
--- a/ingestion/src/test/java/feast/test/TestUtil.java
+++ b/ingestion/src/test/java/feast/test/TestUtil.java
@@ -16,10 +16,17 @@
*/
package feast.test;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
import com.google.protobuf.ByteString;
+import com.google.protobuf.Timestamp;
import com.google.protobuf.util.Timestamps;
+import feast.core.FeatureSetProto.EntitySpec;
import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.core.StoreProto.Store.CassandraConfig;
import feast.ingestion.transform.WriteToStore;
+import feast.ingestion.utils.StoreUtil;
import feast.storage.RedisProto.RedisKey;
import feast.types.FeatureRowProto.FeatureRow;
import feast.types.FeatureRowProto.FeatureRow.Builder;
@@ -33,13 +40,18 @@
import feast.types.ValueProto.StringList;
import feast.types.ValueProto.Value;
import feast.types.ValueProto.ValueType;
+import feast.types.ValueProto.ValueType.Enum;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServerStartable;
import org.apache.beam.sdk.PipelineResult;
@@ -51,8 +63,10 @@
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.LongSerializer;
+import org.apache.thrift.transport.TTransportException;
import org.apache.zookeeper.server.ServerConfig;
import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
import org.joda.time.Duration;
import redis.embedded.RedisServer;
@@ -81,6 +95,37 @@ public static void stop() {
}
}
+ public static class LocalCassandra {
+
+ public static void start() throws InterruptedException, IOException, TTransportException {
+ EmbeddedCassandraServerHelper.startEmbeddedCassandra();
+ }
+
+ public static void createKeyspaceAndTable(CassandraConfig config) {
+ StoreUtil.setupCassandra(config);
+ }
+
+ public static String getHost() {
+ return EmbeddedCassandraServerHelper.getHost();
+ }
+
+ public static int getPort() {
+ return EmbeddedCassandraServerHelper.getNativeTransportPort();
+ }
+
+ public static Cluster getCluster() {
+ return EmbeddedCassandraServerHelper.getCluster();
+ }
+
+ public static Session getSession() {
+ return EmbeddedCassandraServerHelper.getSession();
+ }
+
+ public static void stop() {
+ EmbeddedCassandraServerHelper.cleanEmbeddedCassandra();
+ }
+ }
+
public static class LocalKafka {
private static KafkaServerStartable server;
@@ -163,6 +208,85 @@ public static void publishFeatureRowsToKafka(
});
}
+ /**
+ * Create a Feature Set Spec.
+ *
+ * @param name name of the feature set
+ * @param version version of the feature set
+ * @param maxAgeSeconds max age
+ * @param entities entities provided as map of string to {@link Enum}
+ * @param features features provided as map of string to {@link Enum}
+ * @return {@link FeatureSetSpec}
+ */
+ public static FeatureSetSpec createFeatureSetSpec(
+ String name,
+ int version,
+ int maxAgeSeconds,
+ Map entities,
+ Map features) {
+ FeatureSetSpec.Builder featureSetSpec =
+ FeatureSetSpec.newBuilder()
+ .setName(name)
+ .setVersion(version)
+ .setMaxAge(com.google.protobuf.Duration.newBuilder().setSeconds(maxAgeSeconds).build());
+
+ for (Entry entity : entities.entrySet()) {
+ featureSetSpec.addEntities(
+ EntitySpec.newBuilder().setName(entity.getKey()).setValueType(entity.getValue()).build());
+ }
+
+ for (Entry feature : features.entrySet()) {
+ featureSetSpec.addFeatures(
+ FeatureSpec.newBuilder()
+ .setName(feature.getKey())
+ .setValueType(feature.getValue())
+ .build());
+ }
+
+ return featureSetSpec.build();
+ }
+
+ /**
+ * Create a Feature Row.
+ *
+ * @param featureSetSpec {@link FeatureSetSpec}
+ * @param timestampSeconds timestamp given in seconds
+ * @param fields fields provided as a map name to {@link Value}
+ * @return {@link FeatureRow}
+ */
+ public static FeatureRow createFeatureRow(
+ FeatureSetSpec featureSetSpec, long timestampSeconds, Map fields) {
+ List featureNames =
+ featureSetSpec.getFeaturesList().stream()
+ .map(FeatureSpec::getName)
+ .collect(Collectors.toList());
+ List entityNames =
+ featureSetSpec.getEntitiesList().stream()
+ .map(EntitySpec::getName)
+ .collect(Collectors.toList());
+ List requiredFields =
+ Stream.concat(featureNames.stream(), entityNames.stream()).collect(Collectors.toList());
+
+ if (fields.keySet().containsAll(requiredFields)) {
+ FeatureRow.Builder featureRow =
+ FeatureRow.newBuilder()
+ .setFeatureSet(featureSetSpec.getName() + ":" + featureSetSpec.getVersion())
+ .setEventTimestamp(Timestamp.newBuilder().setSeconds(timestampSeconds).build());
+ for (Entry field : fields.entrySet()) {
+ featureRow.addFields(
+ Field.newBuilder().setName(field.getKey()).setValue(field.getValue()).build());
+ }
+ return featureRow.build();
+ } else {
+ String missingFields =
+ requiredFields.stream()
+ .filter(f -> !fields.keySet().contains(f))
+ .collect(Collectors.joining(","));
+ throw new IllegalArgumentException(
+ "FeatureRow is missing some fields defined in FeatureSetSpec: " + missingFields);
+ }
+ }
+
/**
* Create a Feature Row with random value according to the FeatureSetSpec
*
@@ -352,15 +476,16 @@ public static Field field(String name, Object value, ValueType.Enum valueType) {
/**
* This blocking method waits until an ImportJob pipeline has written all elements to the store.
- *
- * The pipeline must be in the RUNNING state before calling this method.
*
- * @param pipelineResult result of running the Pipeline
+ *
The pipeline must be in the RUNNING state before calling this method.
+ *
+ * @param pipelineResult result of running the Pipeline
* @param maxWaitDuration wait until this max amount of duration
* @throws InterruptedException if the thread is interruped while waiting
*/
- public static void waitUntilAllElementsAreWrittenToStore(PipelineResult pipelineResult,
- Duration maxWaitDuration, Duration checkInterval) throws InterruptedException {
+ public static void waitUntilAllElementsAreWrittenToStore(
+ PipelineResult pipelineResult, Duration maxWaitDuration, Duration checkInterval)
+ throws InterruptedException {
if (pipelineResult.getState().isTerminal()) {
return;
}
@@ -409,4 +534,12 @@ public static void waitUntilAllElementsAreWrittenToStore(PipelineResult pipeline
}
}
}
+
+ public static Value intValue(int val) {
+ return Value.newBuilder().setInt64Val(val).build();
+ }
+
+ public static Value strValue(String val) {
+ return Value.newBuilder().setStringVal(val).build();
+ }
}
diff --git a/ingestion/src/test/proto/DriverArea.proto b/ingestion/src/test/proto/DriverArea.proto
deleted file mode 100644
index fee838b9e17..00000000000
--- a/ingestion/src/test/proto/DriverArea.proto
+++ /dev/null
@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package feast;
-
-option java_outer_classname = "DriverAreaProto";
-
-message DriverArea {
- int32 driverId = 1;
- int32 areaId = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/test/proto/Ping.proto b/ingestion/src/test/proto/Ping.proto
deleted file mode 100644
index b1069afa5bd..00000000000
--- a/ingestion/src/test/proto/Ping.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-syntax = "proto3";
-
-package feast;
-import "google/protobuf/timestamp.proto";
-
-option java_outer_classname = "PingProto";
-
-message Ping {
- double lat = 1;
- double lng = 2;
- google.protobuf.Timestamp timestamp = 3;
-}
diff --git a/pom.xml b/pom.xml
index edf2a0244e9..e352aff2ad7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,12 +22,13 @@
Feature Store for Machine Learning
${github.url}
- feast
+ dev.feast
feast-parent
${revision}
pom
+ datatypes/java
ingestion
core
serving
@@ -35,7 +36,7 @@
- 0.3.2-SNAPSHOT
+ 0.3.7
https://github.com/gojek/feast
UTF-8
@@ -48,7 +49,6 @@
2.16.0
1.91.0
0.8.0
-
1.9.10
1.3
2.3.0
@@ -59,9 +59,18 @@
Gojek
- https://www.gojek.io/
+ https://www.gojek.com
+
+
+ Feast Authors
+ ${github.url}
+ Gojek
+ https://www.gojek.com
+
+
+
Apache License, Version 2.0
@@ -82,15 +91,15 @@
${github.url}/issues
+
-
- feast-snapshot
- file:///tmp/snapshot
+ ossrh
+ https://oss.sonatype.org/content/repositories/snapshots
- feast
- file:///tmp/snapshot
+ ossrh
+ https://oss.sonatype.org/service/local/staging/deploy/maven2/
@@ -132,6 +141,11 @@
+
+ io.grpc
+ grpc-core
+ ${grpcVersion}
+
io.grpc
grpc-netty
@@ -192,7 +206,7 @@
com.google.guava
guava
- 26.0-jre
+ 25.0-jre
com.google.protobuf
@@ -280,10 +294,36 @@
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.2.1
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.1.1
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
com.diffplug.spotless
spotless-maven-plugin
- 1.26.0
+ 1.26.1
@@ -315,6 +355,16 @@
+
+
+
+ spotless-check
+ process-test-classes
+
+ check
+
+
+
org.apache.maven.plugins
@@ -398,6 +448,78 @@
true
+
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.8
+ true
+
+ ossrh
+ https://oss.sonatype.org/
+
+ true
+
+
+
+
+ org.codehaus.mojo
+ flatten-maven-plugin
+ 1.1.0
+
+ oss
+
+
+
+ flatten
+ process-resources
+
+ flatten
+
+
+
+ flatten.clean
+ clean
+
+ clean
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+ 1.6
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+
+
+ --pinentry-mode
+ loopback
+
+
+
+ ${gpg.passphrase}
+
+
+
+
@@ -407,6 +529,20 @@
docker-maven-plugin
0.20.1
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.1.1
+
+
+
+ org.apache.maven.shared
+ maven-dependency-analyzer
+ 1.11.1
+
+
+
org.apache.maven.plugins
maven-javadoc-plugin
@@ -436,25 +572,6 @@
org.xolstice.maven.plugins
protobuf-maven-plugin
0.6.1
-
- true
-
- com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
-
- grpc-java
-
- io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
-
-
-
-
-
- compile
- compile-custom
- test-compile
-
-
-
diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto
index e1b8c581a38..9e1dc33143e 100644
--- a/protos/feast/core/Store.proto
+++ b/protos/feast/core/Store.proto
@@ -17,6 +17,8 @@
syntax = "proto3";
package feast.core;
+import "google/protobuf/duration.proto";
+
option java_package = "feast.core";
option java_outer_classname = "StoreProto";
option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core";
@@ -103,7 +105,20 @@ message Store {
//
BIGQUERY = 2;
- // Unsupported in Feast 0.3
+ // Cassandra stores entities as a string partition key, feature as clustering column.
+ // NOTE: This store currently uses max_age defined in FeatureSet for ttl
+ //
+ // Columns:
+ // - entities: concatenated string of feature set name and all entities' keys and values
+ // entities concatenated format - [feature_set]:[entity_name1=entity_value1]|[entity_name2=entity_value2]
+ // TODO: string representation of float or double types may have different value in different runtime or platform
+ // - feature: clustering column where each feature is a column
+ // - value: byte array of Value (refer to feast.types.Value)
+ //
+ // Internal columns:
+ // - writeTime: timestamp of the written record. This is used to ensure that new records are not replaced
+ // by older ones
+ // - ttl: expiration time the record. Currently using max_age from feature set spec as ttl
CASSANDRA = 3;
}
@@ -118,8 +133,22 @@ message Store {
}
message CassandraConfig {
- string host = 1;
+ // - bootstrapHosts: [comma delimited value of hosts]
+ string bootstrap_hosts = 1;
int32 port = 2;
+ string keyspace = 3;
+
+ // Please note that table name must be "feature_store" as is specified in the @Table annotation of the
+ // datastax object mapper
+ string table_name = 4;
+
+ // This specifies the replication strategy to use. Please refer to docs for more details:
+ // https://docs.datastax.com/en/dse/6.7/cql/cql/cql_reference/cql_commands/cqlCreateKeyspace.html#cqlCreateKeyspace__cqlCreateKeyspacereplicationmap-Pr3yUQ7t
+ map replication_options = 5;
+
+ // Default expiration in seconds to use when FeatureSetSpec does not have max_age defined.
+ // Specify 0 for no default expiration
+ google.protobuf.Duration default_ttl = 6;
}
message Subscription {
diff --git a/sdk/__init__.py b/sdk/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
index 2c8b1d837a3..e8a82a485fc 100644
--- a/sdk/java/pom.xml
+++ b/sdk/java/pom.xml
@@ -6,10 +6,10 @@
Feast SDK for Java
SDK for registering, storing, and retrieving features
- feast-client
+ feast-sdk
- feast
+ dev.feast
feast-parent
${revision}
../..
@@ -21,6 +21,12 @@
+
+ dev.feast
+ datatypes-java
+ ${project.version}
+
+
io.grpc
@@ -79,10 +85,6 @@
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
-
org.apache.maven.plugins
diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java
similarity index 99%
rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java
rename to sdk/java/src/main/java/com/gojek/feast/FeastClient.java
index b7a3e78ab13..91ddd2a442c 100644
--- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java
+++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.gojek.feast.v1alpha1;
+package com.gojek.feast;
import feast.serving.ServingAPIProto.FeatureSetRequest;
import feast.serving.ServingAPIProto.GetFeastServingInfoRequest;
diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java
similarity index 98%
rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java
rename to sdk/java/src/main/java/com/gojek/feast/RequestUtil.java
index 72fbe289f2f..e80b40bad9c 100644
--- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java
+++ b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.gojek.feast.v1alpha1;
+package com.gojek.feast;
import feast.serving.ServingAPIProto.FeatureSetRequest;
import java.util.ArrayList;
diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java
similarity index 99%
rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java
rename to sdk/java/src/main/java/com/gojek/feast/Row.java
index 77f9f298873..9366fe1bb03 100644
--- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java
+++ b/sdk/java/src/main/java/com/gojek/feast/Row.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.gojek.feast.v1alpha1;
+package com.gojek.feast;
import com.google.protobuf.ByteString;
import com.google.protobuf.Timestamp;
diff --git a/sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java
similarity index 99%
rename from sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java
rename to sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java
index 5f87ba01535..21c8bde15ec 100644
--- a/sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java
+++ b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.gojek.feast.v1alpha1;
+package com.gojek.feast;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py
index 3af2e12a91e..0254f4187ba 100644
--- a/sdk/python/feast/client.py
+++ b/sdk/python/feast/client.py
@@ -12,13 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import json
import logging
import os
-import sys
+import time
from collections import OrderedDict
+from math import ceil
from typing import Dict, Union
from typing import List
+from urllib.parse import urlparse
+
+import fastavro
import grpc
import pandas as pd
import pyarrow as pa
@@ -33,11 +37,12 @@
GetFeatureSetResponse,
)
from feast.core.CoreService_pb2_grpc import CoreServiceStub
-from feast.exceptions import format_grpc_exception
from feast.feature_set import FeatureSet, Entity
from feast.job import Job
-from feast.loaders.file import export_dataframe_to_staging_location
-from feast.loaders.ingest import ingest_table_to_kafka
+from feast.loaders.abstract_producer import get_producer
+from feast.loaders.file import export_source_to_staging_location
+from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT
+from feast.loaders.ingest import get_feature_row_chunks
from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse
from feast.serving.ServingService_pb2 import (
GetOnlineFeaturesRequest,
@@ -257,7 +262,7 @@ def _apply_feature_set(self, feature_set: FeatureSet):
print(f"No change detected or applied: {feature_set.name}")
# Deep copy from the returned feature set to the local feature set
- feature_set.update_from_feature_set(applied_fs)
+ feature_set._update_from_feature_set(applied_fs)
def list_feature_sets(self) -> List[FeatureSet]:
"""
@@ -319,22 +324,32 @@ def list_entities(self) -> Dict[str, Entity]:
return entities_dict
def get_batch_features(
- self, feature_ids: List[str], entity_rows: pd.DataFrame
+ self, feature_ids: List[str], entity_rows: Union[pd.DataFrame, str]
) -> Job:
"""
Retrieves historical features from a Feast Serving deployment.
Args:
- feature_ids: List of feature ids that will be returned for each
- entity. Each feature id should have the following format
+ feature_ids (List[str]):
+ List of feature ids that will be returned for each entity.
+ Each feature id should have the following format
"feature_set_name:version:feature_name".
- entity_rows: Pandas dataframe containing entities and a 'datetime'
- column. Each entity in a feature set must be present as a column
- in this dataframe. The datetime column must
+
+ entity_rows (Union[pd.DataFrame, str]):
+ Either:
+ Pandas dataframe containing entities and a 'datetime' column.
+ Each entity in a feature set must be present as a column in this
+ dataframe. The datetime column must contain timestamps in
+ datetime64 format.
+
+ Or:
+ A file path in AVRO format representing the entity rows.
Returns:
- Returns a job object that can be used to monitor retrieval progress
- asynchronously, and can be used to materialize the results
+ feast.job.Job:
+ Returns a job object that can be used to monitor retrieval
+ progress asynchronously, and can be used to materialize the
+ results.
Examples:
>>> from feast import Client
@@ -357,21 +372,11 @@ def get_batch_features(
fs_request = _build_feature_set_request(feature_ids)
- # Validate entity rows based on entities in Feast Core
- self._validate_entity_rows_for_batch_retrieval(entity_rows, fs_request)
-
- # Remove timezone from datetime column
- if isinstance(
- entity_rows["datetime"].dtype, pd.core.dtypes.dtypes.DatetimeTZDtype
- ):
- entity_rows["datetime"] = pd.DatetimeIndex(
- entity_rows["datetime"]
- ).tz_localize(None)
-
# Retrieve serving information to determine store type and
# staging location
serving_info = self._serving_service_stub.GetFeastServingInfo(
- GetFeastServingInfoRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT
+ GetFeastServingInfoRequest(),
+ timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT
) # type: GetFeastServingInfoResponse
if serving_info.type != FeastServingType.FEAST_SERVING_TYPE_BATCH:
@@ -380,17 +385,50 @@ def get_batch_features(
f"does not support batch retrieval "
)
- # Export and upload entity row dataframe to staging location
+ if isinstance(entity_rows, pd.DataFrame):
+ # Pandas DataFrame detected
+ # Validate entity rows to based on entities in Feast Core
+ self._validate_dataframe_for_batch_retrieval(
+ entity_rows=entity_rows,
+ feature_sets_request=fs_request
+ )
+
+ # Remove timezone from datetime column
+ if isinstance(
+ entity_rows["datetime"].dtype,
+ pd.core.dtypes.dtypes.DatetimeTZDtype
+ ):
+ entity_rows["datetime"] = pd.DatetimeIndex(
+ entity_rows["datetime"]
+ ).tz_localize(None)
+ elif isinstance(entity_rows, str):
+ # String based source
+ if entity_rows.endswith((".avro", "*")):
+ # Validate Avro entity rows to based on entities in Feast Core
+ self._validate_avro_for_batch_retrieval(
+ source=entity_rows,
+ feature_sets_request=fs_request
+ )
+ else:
+ raise Exception(
+ f"Only .avro and wildcard paths are accepted as entity_rows"
+ )
+ else:
+ raise Exception(f"Only pandas.DataFrame and str types are allowed"
+ f" as entity_rows, but got {type(entity_rows)}.")
+
+ # Export and upload entity row DataFrame to staging location
# provided by Feast
- staged_file = export_dataframe_to_staging_location(
+ staged_files = export_source_to_staging_location(
entity_rows, serving_info.job_staging_location
- ) # type: str
+ ) # type: List[str]
request = GetBatchFeaturesRequest(
feature_sets=fs_request,
dataset_source=DatasetSource(
file_source=DatasetSource.FileSource(
- file_uris=[staged_file], data_format=DataFormat.DATA_FORMAT_AVRO
+ file_uris=staged_files,
+ data_format=DataFormat.DATA_FORMAT_AVRO
)
),
)
@@ -399,28 +437,107 @@ def get_batch_features(
response = self._serving_service_stub.GetBatchFeatures(request)
return Job(response.job, self._serving_service_stub)
- def _validate_entity_rows_for_batch_retrieval(
- self, entity_rows, feature_sets_request
+ def _validate_dataframe_for_batch_retrieval(
+ self, entity_rows: pd.DataFrame, feature_sets_request
):
"""
- Validate whether an entity_row dataframe contains the correct
- information for batch retrieval
+ Validate whether an the entity rows in a DataFrame contains the correct
+ information for batch retrieval.
+
+ Datetime column must be present in the DataFrame.
Args:
- entity_rows: Pandas dataframe containing entities and datetime
- column. Each entity in a feature set must be present as a
- column in this dataframe.
- feature_sets_request: Feature sets that will be requested
+ entity_rows (pd.DataFrame):
+ Pandas DataFrame containing entities and datetime column. Each
+ entity in a feature set must be present as a column in this
+ DataFrame.
+
+ feature_sets_request:
+ Feature sets that will be requested.
"""
+ self._validate_columns(
+ columns=entity_rows.columns,
+ feature_sets_request=feature_sets_request,
+ datetime_field="datetime"
+ )
+
+ def _validate_avro_for_batch_retrieval(
+ self, source: str, feature_sets_request
+ ):
+ """
+ Validate whether the entity rows in an Avro source file contains the
+ correct information for batch retrieval.
+
+ Only gs:// and local files (file://) uri schemes are allowed.
+
+ Avro file must have a column named "event_timestamp".
+
+ No checks will be done if a GCS path is provided.
+
+ Args:
+ source (str):
+ File path to Avro.
+
+ feature_sets_request:
+ Feature sets that will be requested.
+ """
+ p = urlparse(source)
+
+ if p.scheme == "gs":
+ # GCS path provided (Risk is delegated to user)
+ # No validation if GCS path is provided
+ return
+ elif p.scheme == "file" or not p.scheme:
+ # Local file (file://) provided
+ file_path = os.path.abspath(os.path.join(p.netloc, p.path))
+ else:
+ raise Exception(f"Unsupported uri scheme provided {p.scheme}, only "
+ f"local files (file://), and gs:// schemes are "
+ f"allowed")
+
+ with open(file_path, "rb") as f:
+ reader = fastavro.reader(f)
+ schema = json.loads(reader.metadata["avro.schema"])
+ columns = [x["name"] for x in schema["fields"]]
+ self._validate_columns(
+ columns=columns,
+ feature_sets_request=feature_sets_request,
+ datetime_field="event_timestamp"
+ )
+
+ def _validate_columns(
+ self, columns: List[str],
+ feature_sets_request,
+ datetime_field: str
+ ) -> None:
+ """
+ Check if the required column contains the correct values for batch
+ retrieval.
+
+ Args:
+ columns (List[str]):
+ List of columns to validate against feature_sets_request.
+
+ feature_sets_request ():
+ Feature sets that will be requested.
+
+ datetime_field (str):
+ Name of the datetime field that must be enforced and present as
+ a column in the data source.
+
+ Returns:
+ None:
+ None
+ """
# Ensure datetime column exists
- if "datetime" not in entity_rows.columns:
+ if datetime_field not in columns:
raise ValueError(
- f'Entity rows does not contain "datetime" column in columns '
- f"{entity_rows.columns}"
+ f'Entity rows does not contain "{datetime_field}" column in '
+ f'columns {columns}'
)
- # Validate dataframe columns based on feature set entities
+ # Validate Avro columns based on feature set entities
for feature_set in feature_sets_request:
fs = self.get_feature_set(
name=feature_set.name, version=feature_set.version
@@ -431,10 +548,10 @@ def _validate_entity_rows_for_batch_retrieval(
f"could not be found"
)
for entity_type in fs.entities:
- if entity_type.name not in entity_rows.columns:
+ if entity_type.name not in columns:
raise ValueError(
- f'Dataframe does not contain entity "{entity_type.name}"'
- f' column in columns "{entity_rows.columns}"'
+ f'Input does not contain entity'
+ f' "{entity_type.name}" column in columns "{columns}"'
)
def get_online_features(
@@ -470,35 +587,55 @@ def get_online_features(
) # type: GetOnlineFeaturesResponse
def ingest(
- self,
- feature_set: Union[str, FeatureSet],
- source: Union[pd.DataFrame, str],
- version: int = None,
- force_update: bool = False,
- max_workers: int = CPU_COUNT,
- disable_progress_bar: bool = False,
- chunk_size: int = 5000,
- timeout: int = None,
- ):
+ self,
+ feature_set: Union[str, FeatureSet],
+ source: Union[pd.DataFrame, str],
+ chunk_size: int = 10000,
+ version: int = None,
+ force_update: bool = False,
+ max_workers: int = max(CPU_COUNT - 1, 1),
+ disable_progress_bar: bool = False,
+ timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT
+ ) -> None:
"""
Loads feature data into Feast for a specific feature set.
Args:
- feature_set: Name of feature set or a feature set object
- source: Either a file path or Pandas Dataframe to ingest into Feast
+ feature_set (typing.Union[str, FeatureSet]):
+ Feature set object or the string name of the feature set
+ (without a version).
+
+ source (typing.Union[pd.DataFrame, str]):
+ Either a file path or Pandas Dataframe to ingest into Feast
Files that are currently supported:
- * parquet
- * csv
- * json
- version: Feature set version
- force_update: Automatically update feature set based on source data
- prior to ingesting. This will also register changes to Feast
- max_workers: Number of worker processes to use to encode values
- disable_progress_bar: Disable printing of progress statistics
- chunk_size: Maximum amount of rows to load into memory and ingest at
- a time
- timeout: Seconds to wait before ingestion times out
+ * parquet
+ * csv
+ * json
+
+ chunk_size (int):
+ Amount of rows to load and ingest at a time.
+
+ version (int):
+ Feature set version.
+
+ force_update (bool):
+ Automatically update feature set based on source data prior to
+ ingesting. This will also register changes to Feast.
+
+ max_workers (int):
+ Number of worker processes to use to encode values.
+
+ disable_progress_bar (bool):
+ Disable printing of progress statistics.
+
+ timeout (int):
+ Timeout in seconds to wait for completion.
+
+ Returns:
+ None:
+ None
"""
+
if isinstance(feature_set, FeatureSet):
name = feature_set.name
if version is None:
@@ -508,38 +645,74 @@ def ingest(
else:
raise Exception(f"Feature set name must be provided")
- table = _read_table_from_source(source)
+ # Read table and get row count
+ tmp_table_name = _read_table_from_source(
+ source, chunk_size, max_workers
+ )
+
+ pq_file = pq.ParquetFile(tmp_table_name)
- # Update the feature set based on DataFrame schema
- if force_update:
- # Use a small as reference DataFrame to infer fields
- ref_df = table.to_batches(max_chunksize=20)[0].to_pandas()
+ row_count = pq_file.metadata.num_rows
- feature_set.infer_fields_from_df(
- ref_df, discard_unused_fields=True, replace_existing_features=True
+ # Update the feature set based on PyArrow table of first row group
+ if force_update:
+ feature_set.infer_fields_from_pa(
+ table=pq_file.read_row_group(0),
+ discard_unused_fields=True,
+ replace_existing_features=True
)
self.apply(feature_set)
feature_set = self.get_feature_set(name, version)
- if feature_set.source.source_type == "Kafka":
- ingest_table_to_kafka(
- feature_set=feature_set,
- table=table,
- max_workers=max_workers,
- disable_pbar=disable_progress_bar,
- chunk_size=chunk_size,
- timeout=timeout,
- )
- else:
- raise Exception(
- f"Could not determine source type for feature set "
- f'"{feature_set.name}" with source type '
- f'"{feature_set.source.source_type}"'
- )
+ try:
+ # Kafka configs
+ brokers = feature_set.get_kafka_source_brokers()
+ topic = feature_set.get_kafka_source_topic()
+ producer = get_producer(brokers, row_count, disable_progress_bar)
+
+ # Loop optimization declarations
+ produce = producer.produce
+ flush = producer.flush
+
+ # Transform and push data to Kafka
+ if feature_set.source.source_type == "Kafka":
+ for chunk in get_feature_row_chunks(
+ file=tmp_table_name,
+ row_groups=list(range(pq_file.num_row_groups)),
+ fs=feature_set,
+ max_workers=max_workers):
+
+ # Push FeatureRow one chunk at a time to kafka
+ for serialized_row in chunk:
+ produce(topic=topic, value=serialized_row)
+
+ # Force a flush after each chunk
+ flush(timeout=timeout)
+
+ # Remove chunk from memory
+ del chunk
+
+ else:
+ raise Exception(
+ f"Could not determine source type for feature set "
+ f'"{feature_set.name}" with source type '
+ f'"{feature_set.source.source_type}"'
+ )
+
+ # Print ingestion statistics
+ producer.print_results()
+ finally:
+ # Remove parquet file(s) that were created earlier
+ print("Removing temporary file(s)...")
+ os.remove(tmp_table_name)
+
+ return None
-def _build_feature_set_request(feature_ids: List[str]) -> List[FeatureSetRequest]:
+def _build_feature_set_request(
+ feature_ids: List[str]
+) -> List[FeatureSetRequest]:
"""
Builds a list of FeatureSet objects from feature set ids in order to
retrieve feature data from Feast Serving
@@ -566,18 +739,38 @@ def _build_feature_set_request(feature_ids: List[str]) -> List[FeatureSetRequest
return list(feature_set_request.values())
-def _read_table_from_source(source: Union[pd.DataFrame, str]) -> pa.lib.Table:
+def _read_table_from_source(
+ source: Union[pd.DataFrame, str],
+ chunk_size: int,
+ max_workers: int
+) -> str:
"""
- Infers a data source type (path or Pandas Dataframe) and reads it in as
+ Infers a data source type (path or Pandas DataFrame) and reads it in as
a PyArrow Table.
+ The PyArrow Table that is read will be written to a parquet file with row
+ group size determined by the minimum of:
+ * (table.num_rows / max_workers)
+ * chunk_size
+
+ The parquet file that is created will be passed as file path to the
+ multiprocessing pool workers.
+
Args:
- source: Either a string path or Pandas Dataframe
+ source (Union[pd.DataFrame, str]):
+ Either a string path or Pandas DataFrame.
+
+ chunk_size (int):
+ Number of worker processes to use to encode values.
+
+ max_workers (int):
+ Amount of rows to load and ingest at a time.
Returns:
- PyArrow table
+ str: Path to parquet file that was created.
"""
- # Pandas dataframe detected
+
+ # Pandas DataFrame detected
if isinstance(source, pd.DataFrame):
table = pa.Table.from_pandas(df=source)
@@ -597,8 +790,19 @@ def _read_table_from_source(source: Union[pd.DataFrame, str]) -> pa.lib.Table:
else:
table = pq.read_table(file_path)
else:
- raise ValueError(f"Unknown data source provided for ingestion: {source}")
+ raise ValueError(
+ f"Unknown data source provided for ingestion: {source}")
# Ensure that PyArrow table is initialised
assert isinstance(table, pa.lib.Table)
- return table
+
+ # Write table as parquet file with a specified row_group_size
+ tmp_table_name = f"{int(time.time())}.parquet"
+ row_group_size = min(ceil(table.num_rows / max_workers), chunk_size)
+ pq.write_table(table=table, where=tmp_table_name,
+ row_group_size=row_group_size)
+
+ # Remove table from memory
+ del table
+
+ return tmp_table_name
diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py
index 893378e8fac..42979ea9116 100644
--- a/sdk/python/feast/feature_set.py
+++ b/sdk/python/feast/feature_set.py
@@ -13,21 +13,25 @@
# limitations under the License.
-import pandas as pd
-from typing import List, Optional
from collections import OrderedDict
from typing import Dict
-from feast.source import Source
-from pandas.api.types import is_datetime64_ns_dtype
+from typing import List, Optional
+
+import pandas as pd
+import pyarrow as pa
+from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto
from feast.entity import Entity
from feast.feature import Feature, Field
-from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto
-from google.protobuf.duration_pb2 import Duration
+from feast.loaders import yaml as feast_yaml
+from feast.source import Source
+from feast.type_map import DATETIME_COLUMN
+from feast.type_map import pa_to_feast_value_type
from feast.type_map import python_type_to_feast_value_type
-from google.protobuf.json_format import MessageToJson
from google.protobuf import json_format
-from feast.type_map import DATETIME_COLUMN
-from feast.loaders import yaml as feast_yaml
+from google.protobuf.duration_pb2 import Duration
+from google.protobuf.json_format import MessageToJson
+from pandas.api.types import is_datetime64_ns_dtype
+from pyarrow.lib import TimestampType
class FeatureSet:
@@ -256,7 +260,6 @@ def infer_fields_from_df(
rows_to_sample: int = 100,
):
"""
-
Adds fields (Features or Entities) to a feature set based on the schema
of a Datatframe. Only Pandas dataframes are supported. All columns are
detected as features, so setting at least one entity manually is
@@ -283,6 +286,7 @@ def infer_fields_from_df(
must have consistent types, even values within list types must
be homogeneous
"""
+
if entities is None:
entities = list()
if features is None:
@@ -373,7 +377,187 @@ def infer_fields_from_df(
self._fields = new_fields
print(output_log)
- def update_from_feature_set(self, feature_set):
+ def infer_fields_from_pa(
+ self, table: pa.lib.Table,
+ entities: Optional[List[Entity]] = None,
+ features: Optional[List[Feature]] = None,
+ replace_existing_features: bool = False,
+ replace_existing_entities: bool = False,
+ discard_unused_fields: bool = False
+ ) -> None:
+ """
+ Adds fields (Features or Entities) to a feature set based on the schema
+ of a PyArrow table. Only PyArrow tables are supported. All columns are
+ detected as features, so setting at least one entity manually is
+ advised.
+
+
+ Args:
+ table (pyarrow.lib.Table):
+ PyArrow table to read schema from.
+
+ entities (Optional[List[Entity]]):
+ List of entities that will be set manually and not inferred.
+ These will take precedence over any existing entities or
+ entities found in the PyArrow table.
+
+ features (Optional[List[Feature]]):
+ List of features that will be set manually and not inferred.
+ These will take precedence over any existing feature or features
+ found in the PyArrow table.
+
+ replace_existing_features (bool):
+ Boolean flag. If true, will replace existing features in this
+ feature set with features found in dataframe. If false, will
+ skip conflicting features.
+
+ replace_existing_entities (bool):
+ Boolean flag. If true, will replace existing entities in this
+ feature set with features found in dataframe. If false, will
+ skip conflicting entities.
+
+ discard_unused_fields (bool):
+ Boolean flag. Setting this to True will discard any existing
+ fields that are not found in the dataset or provided by the
+ user.
+
+ Returns:
+ None:
+ None
+ """
+ if entities is None:
+ entities = list()
+ if features is None:
+ features = list()
+
+ # Validate whether the datetime column exists with the right name
+ if DATETIME_COLUMN not in table.column_names:
+ raise Exception("No column 'datetime'")
+
+ # Validate the date type for the datetime column
+ if not isinstance(table.column(DATETIME_COLUMN).type, TimestampType):
+ raise Exception(
+ "Column 'datetime' does not have the correct type: datetime64[ms]"
+ )
+
+ # Create dictionary of fields that will not be inferred (manually set)
+ provided_fields = OrderedDict()
+
+ for field in entities + features:
+ if not isinstance(field, Field):
+ raise Exception(f"Invalid field object type provided {type(field)}")
+ if field.name not in provided_fields:
+ provided_fields[field.name] = field
+ else:
+ raise Exception(f"Duplicate field name detected {field.name}.")
+
+ new_fields = self._fields.copy()
+ output_log = ""
+
+ # Add in provided fields
+ for name, field in provided_fields.items():
+ if name in new_fields.keys():
+ upsert_message = "created"
+ else:
+ upsert_message = "updated (replacing an existing field)"
+
+ output_log += (
+ f"{type(field).__name__} {field.name}"
+ f"({field.dtype}) manually {upsert_message}.\n"
+ )
+ new_fields[name] = field
+
+ # Iterate over all of the column names and create features
+ for column in table.column_names:
+ column = column.strip()
+
+ # Skip datetime column
+ if DATETIME_COLUMN in column:
+ continue
+
+ # Skip user provided fields
+ if column in provided_fields.keys():
+ continue
+
+ # Only overwrite conflicting fields if replacement is allowed
+ if column in new_fields:
+ if (
+ isinstance(self._fields[column], Feature)
+ and not replace_existing_features
+ ):
+ continue
+
+ if (
+ isinstance(self._fields[column], Entity)
+ and not replace_existing_entities
+ ):
+ continue
+
+ # Store this fields as a feature
+ # TODO: (Minor) Change the parameter name from dtype to patype
+ new_fields[column] = Feature(
+ name=column,
+ dtype=self._infer_pa_column_type(table.column(column))
+ )
+
+ output_log += f"{type(new_fields[column]).__name__} {new_fields[column].name} ({new_fields[column].dtype}) added from PyArrow Table.\n"
+
+ # Discard unused fields from feature set
+ if discard_unused_fields:
+ keys_to_remove = []
+ for key in new_fields.keys():
+ if not (key in table.column_names or key in provided_fields.keys()):
+ output_log += f"{type(new_fields[key]).__name__} {new_fields[key].name} ({new_fields[key].dtype}) removed because it is unused.\n"
+ keys_to_remove.append(key)
+ for key in keys_to_remove:
+ del new_fields[key]
+
+ # Update feature set
+ self._fields = new_fields
+ print(output_log)
+
+ def _infer_pd_column_type(self, column, series, rows_to_sample):
+ dtype = None
+ sample_count = 0
+
+ # Loop over all rows for this column to infer types
+ for key, value in series.iteritems():
+ sample_count += 1
+ # Stop sampling at the row limit
+ if sample_count > rows_to_sample:
+ continue
+
+ # Infer the specific type for this row
+ current_dtype = python_type_to_feast_value_type(name=column, value=value)
+
+ # Make sure the type is consistent for column
+ if dtype:
+ if dtype != current_dtype:
+ raise ValueError(
+ f"Type mismatch detected in column {column}. Both "
+ f"the types {current_dtype} and {dtype} "
+ f"have been found."
+ )
+ else:
+ # Store dtype in field to type map if it isnt already
+ dtype = current_dtype
+
+ return dtype
+
+ def _infer_pa_column_type(self, column: pa.lib.ChunkedArray):
+ """
+ Infers the PyArrow column type.
+
+ :param column: Column from a PyArrow table
+ :type column: pa.lib.ChunkedArray
+ :return:
+ :rtype:
+ """
+ # Validates the column to ensure that value types are consistent
+ column.validate()
+ return pa_to_feast_value_type(column)
+
+ def _update_from_feature_set(self, feature_set):
"""
Deep replaces one feature set with another
diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py
index 4273f86ea84..26f6181ee2d 100644
--- a/sdk/python/feast/job.py
+++ b/sdk/python/feast/job.py
@@ -1,12 +1,11 @@
import tempfile
import time
from datetime import datetime, timedelta
-from typing import List
+from typing import Iterable
from urllib.parse import urlparse
import fastavro
import pandas as pd
-from fastavro import reader as fastavro_reader
from google.cloud import storage
from feast.serving.ServingService_pb2 import GetJobRequest
@@ -62,15 +61,18 @@ def reload(self):
"""
self.job_proto = self.serving_stub.GetJob(GetJobRequest(job=self.job_proto)).job
- def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
+ def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
"""
- Wait until job is done to get an iterable rows of result.
- The row can only represent an Avro row in Feast 0.3.
+ Wait until job is done to get the file uri to Avro result files on
+ Google Cloud Storage.
Args:
- timeout_sec: max no of seconds to wait until job is done. If "timeout_sec" is exceeded, an exception will be raised.
+ timeout_sec (int):
+ Max no of seconds to wait until job is done. If "timeout_sec"
+ is exceeded, an exception will be raised.
- Returns: Iterable of Avro rows
+ Returns:
+ str: Google Cloud Storage file uris of the returned Avro files.
"""
max_wait_datetime = datetime.now() + timedelta(seconds=timeout_sec)
wait_duration_sec = 2
@@ -78,11 +80,13 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
while self.status != JOB_STATUS_DONE:
if datetime.now() > max_wait_datetime:
raise Exception(
- "Timeout exceeded while waiting for result. Please retry this method or use a longer timeout value."
+ "Timeout exceeded while waiting for result. Please retry "
+ "this method or use a longer timeout value."
)
self.reload()
time.sleep(wait_duration_sec)
+
# Backoff the wait duration exponentially up till MAX_WAIT_INTERVAL_SEC
wait_duration_sec = min(wait_duration_sec * 2, MAX_WAIT_INTERVAL_SEC)
@@ -95,7 +99,22 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
"your Feast Serving deployment."
)
- uris = [urlparse(uri) for uri in self.job_proto.file_uris]
+ return [urlparse(uri) for uri in self.job_proto.file_uris]
+
+ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
+ """
+ Wait until job is done to get an iterable rows of result. The row can
+ only represent an Avro row in Feast 0.3.
+
+ Args:
+ timeout_sec (int):
+ Max no of seconds to wait until job is done. If "timeout_sec"
+ is exceeded, an exception will be raised.
+
+ Returns:
+ Iterable of Avro rows.
+ """
+ uris = self.get_avro_files(timeout_sec)
for file_uri in uris:
if file_uri.scheme == "gs":
file_obj = tempfile.TemporaryFile()
@@ -113,16 +132,64 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
for record in avro_reader:
yield record
- def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
+ def to_dataframe(
+ self,
+ timeout_sec: int = DEFAULT_TIMEOUT_SEC
+ ) -> pd.DataFrame:
"""
- Wait until job is done to get an interable rows of result
+ Wait until a job is done to get an iterable rows of result. This method
+ will split the response into chunked DataFrame of a specified size to
+ to be yielded to the instance calling it.
Args:
- timeout_sec: max no of seconds to wait until job is done. If "timeout_sec" is exceeded, an exception will be raised.
- Returns: pandas Dataframe of the feature values
+ max_chunk_size (int):
+ Maximum number of rows that the DataFrame should contain.
+
+ timeout_sec (int):
+ Max no of seconds to wait until job is done. If "timeout_sec"
+ is exceeded, an exception will be raised.
+
+ Returns:
+ pd.DataFrame:
+ Pandas DataFrame of the feature values.
"""
records = [r for r in self.result(timeout_sec=timeout_sec)]
return pd.DataFrame.from_records(records)
+ def to_chunked_dataframe(
+ self,
+ max_chunk_size: int = -1,
+ timeout_sec: int = DEFAULT_TIMEOUT_SEC
+ ) -> pd.DataFrame:
+ """
+ Wait until a job is done to get an iterable rows of result. This method
+ will split the response into chunked DataFrame of a specified size to
+ to be yielded to the instance calling it.
+
+ Args:
+ max_chunk_size (int):
+ Maximum number of rows that the DataFrame should contain.
+
+ timeout_sec (int):
+ Max no of seconds to wait until job is done. If "timeout_sec"
+ is exceeded, an exception will be raised.
+
+ Returns:
+ pd.DataFrame:
+ Pandas DataFrame of the feature values.
+ """
+ # Max chunk size defined by user
+ records = []
+ for result in self.result(timeout_sec=timeout_sec):
+ result.append(records)
+ if len(records) == max_chunk_size:
+ df = pd.DataFrame.from_records(records)
+ records.clear() # Empty records array
+ yield df
+
+ # Handle for last chunk that is < max_chunk_size
+ if not records:
+ yield pd.DataFrame.from_records(records)
+
def __iter__(self):
return iter(self.result())
diff --git a/sdk/python/feast/loaders/abstract_producer.py b/sdk/python/feast/loaders/abstract_producer.py
new file mode 100644
index 00000000000..884ae49984c
--- /dev/null
+++ b/sdk/python/feast/loaders/abstract_producer.py
@@ -0,0 +1,248 @@
+# Copyright 2019 The Feast Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Union
+
+from tqdm import tqdm
+
+
+class AbstractProducer:
+ """
+ Abstract class for Kafka producers
+ """
+
+ def __init__(
+ self,
+ brokers: str,
+ row_count: int,
+ disable_progress_bar: bool
+ ):
+ self.brokers = brokers
+ self.row_count = row_count
+ self.error_count = 0
+ self.last_exception = ""
+
+ # Progress bar will always display average rate
+ self.pbar = tqdm(
+ total=row_count,
+ unit="rows",
+ smoothing=0,
+ disable=disable_progress_bar
+ )
+
+ def produce(self, topic: str, data: str):
+ message = "{} should implement a produce method".format(
+ self.__class__.__name__)
+ raise NotImplementedError(message)
+
+ def flush(self, timeout: int):
+ message = "{} should implement a flush method".format(
+ self.__class__.__name__)
+ raise NotImplementedError(message)
+
+ def _inc_pbar(self, meta):
+ self.pbar.update(1)
+
+ def _set_error(self, exception: str):
+ self.error_count += 1
+ self.last_exception = exception
+
+ def print_results(self) -> None:
+ """
+ Print ingestion statistics.
+
+ Returns:
+ None: None
+ """
+ # Refresh and close tqdm progress bar
+ self.pbar.refresh()
+
+ self.pbar.close()
+
+ print("Ingestion complete!")
+
+ failed_message = (
+ ""
+ if self.error_count == 0
+ else f"\nFail: {self.error_count / self.row_count}"
+ )
+
+ last_exception_message = (
+ ""
+ if self.last_exception == ""
+ else f"\nLast exception:\n{self.last_exception}"
+ )
+
+ print(
+ f"\nIngestion statistics:"
+ f"\nSuccess: {self.pbar.n}/{self.row_count}"
+ f"{failed_message}"
+ f"{last_exception_message}"
+ )
+ return None
+
+
+class ConfluentProducer(AbstractProducer):
+ """
+ Concrete implementation of Confluent Kafka producer (confluent-kafka)
+ """
+
+ def __init__(
+ self,
+ brokers: str,
+ row_count: int,
+ disable_progress_bar: bool
+ ):
+ from confluent_kafka import Producer
+ self.producer = Producer({"bootstrap.servers": brokers})
+ super().__init__(brokers, row_count, disable_progress_bar)
+
+ def produce(self, topic: str, value: bytes) -> None:
+ """
+ Generic produce that implements confluent-kafka's produce method to
+ push a byte encoded object into a Kafka topic.
+
+ Args:
+ topic (str): Kafka topic.
+ value (bytes): Byte encoded object.
+
+ Returns:
+ None: None.
+ """
+
+ try:
+ self.producer.produce(
+ topic, value=value, callback=self._delivery_callback)
+ # Serve delivery callback queue.
+ # NOTE: Since produce() is an asynchronous API this poll() call
+ # will most likely not serve the delivery callback for the
+ # last produce()d message.
+ self.producer.poll(0)
+ except Exception as ex:
+ self._set_error(str(ex))
+
+ return None
+
+ def flush(self, timeout: Optional[int]):
+ """
+ Generic flush that implements confluent-kafka's flush method.
+
+ Args:
+ timeout (Optional[int]): Timeout in seconds to wait for completion.
+
+ Returns:
+ int: Number of messages still in queue.
+ """
+ return self.producer.flush(timeout=timeout)
+
+ def _delivery_callback(self, err: str, msg) -> None:
+ """
+ Optional per-message delivery callback (triggered by poll() or flush())
+ when a message has been successfully delivered or permanently failed
+ delivery (after retries).
+
+ Although the msg argument is not used, the current method signature is
+ required as specified in the confluent-kafka documentation.
+
+ Args:
+ err (str): Error message.
+ msg (): Kafka message.
+
+ Returns:
+ None
+ """
+ if err:
+ self._set_error(err)
+ else:
+ self._inc_pbar(None)
+
+
+class KafkaPythonProducer(AbstractProducer):
+ """
+ Concrete implementation of Python Kafka producer (kafka-python)
+ """
+
+ def __init__(
+ self,
+ brokers: str,
+ row_count: int,
+ disable_progress_bar: bool
+ ):
+ from kafka import KafkaProducer
+ self.producer = KafkaProducer(bootstrap_servers=[brokers])
+ super().__init__(brokers, row_count, disable_progress_bar)
+
+ def produce(self, topic: str, value: bytes):
+ """
+ Generic produce that implements kafka-python's send method to push a
+ byte encoded object into a Kafka topic.
+
+ Args:
+ topic (str): Kafka topic.
+ value (bytes): Byte encoded object.
+
+ Returns:
+ FutureRecordMetadata: resolves to RecordMetadata
+
+ Raises:
+ KafkaTimeoutError: if unable to fetch topic metadata, or unable
+ to obtain memory buffer prior to configured max_block_ms
+ """
+ return self.producer.send(topic, value=value).add_callback(
+ self._inc_pbar).add_errback(self._set_error)
+
+ def flush(self, timeout: Optional[int]):
+ """
+ Generic flush that implements kafka-python's flush method.
+
+ Args:
+ timeout (Optional[int]): timeout in seconds to wait for completion.
+
+ Returns:
+ None
+
+ Raises:
+ KafkaTimeoutError: failure to flush buffered records within the
+ provided timeout
+ """
+ return self.producer.flush(timeout=timeout)
+
+
+def get_producer(
+ brokers: str, row_count: int, disable_progress_bar: bool
+) -> Union[ConfluentProducer, KafkaPythonProducer]:
+ """
+ Simple context helper function that returns a AbstractProducer object when
+ invoked.
+
+ This helper function will try to import confluent-kafka as a producer first.
+
+ This helper function will fallback to kafka-python if it fails to import
+ confluent-kafka.
+
+ Args:
+ brokers (str): Kafka broker information with hostname and port.
+ row_count (int): Number of rows in table
+
+ Returns:
+ Union[ConfluentProducer, KafkaPythonProducer]:
+ Concrete implementation of a Kafka producer. Ig can be:
+ * confluent-kafka producer
+ * kafka-python producer
+ """
+ try:
+ return ConfluentProducer(brokers, row_count, disable_progress_bar)
+ except ImportError as e:
+ print("Unable to import confluent-kafka, falling back to kafka-python")
+ return KafkaPythonProducer(brokers, row_count, disable_progress_bar)
diff --git a/sdk/python/feast/loaders/file.py b/sdk/python/feast/loaders/file.py
index 8dd6b503a74..108f2790dd8 100644
--- a/sdk/python/feast/loaders/file.py
+++ b/sdk/python/feast/loaders/file.py
@@ -1,70 +1,159 @@
+# Copyright 2019 The Feast Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
import shutil
import tempfile
-from typing import Optional
-from urllib.parse import urlparse
import uuid
-import pandas as pd
from datetime import datetime
+from typing import List, Optional, Tuple, Union
+from urllib.parse import urlparse, ParseResult
+
+import pandas as pd
from google.cloud import storage
from pandavro import to_avro
-def export_dataframe_to_staging_location(
- df: pd.DataFrame, staging_location_uri: str
-) -> str:
+def export_source_to_staging_location(
+ source: Union[pd.DataFrame, str], staging_location_uri: str
+) -> List[str]:
"""
- Uploads a dataframe to a remote staging location
+ Uploads a DataFrame as an Avro file to a remote staging location.
+
+ The local staging location specified in this function is used for E2E
+ tests, please do not use it.
Args:
- df: Pandas dataframe
- staging_location_uri: Remote staging location where dataframe should be written
+ source (Union[pd.DataFrame, str]:
+ Source of data to be staged. Can be a pandas DataFrame or a file
+ path.
+
+ Only three types of source are allowed:
+ * Pandas DataFrame
+ * Local Avro file
+ * GCS Avro file
+
+
+ staging_location_uri (str):
+ Remote staging location where DataFrame should be written.
Examples:
- gs://bucket/path/
- file:///data/subfolder/
+ * gs://bucket/path/
+ * file:///data/subfolder/
Returns:
- Returns the full path to the file in the remote staging location
+ List[str]:
+ Returns a list containing the full path to the file(s) in the
+ remote staging location.
"""
- # Validate staging location
uri = urlparse(staging_location_uri)
+
+ # Prepare Avro file to be exported to staging location
+ if isinstance(source, pd.DataFrame):
+ # DataFrame provided as a source
+ if uri.scheme == "file":
+ uri_path = uri.path
+ else:
+ uri_path = None
+
+ # Remote gs staging location provided by serving
+ dir_path, file_name, source_path = export_dataframe_to_local(
+ source,
+ uri_path
+ )
+ elif urlparse(source).scheme in ["", "file"]:
+ # Local file provided as a source
+ dir_path = None
+ file_name = os.path.basename(source)
+ source_path = os.path.abspath(os.path.join(
+ urlparse(source).netloc, urlparse(source).path))
+ elif urlparse(source).scheme == "gs":
+ # Google Cloud Storage path provided
+ input_source_uri = urlparse(source)
+ if "*" in source:
+ # Wildcard path
+ return _get_files(
+ bucket=input_source_uri.hostname,
+ uri=input_source_uri
+ )
+ else:
+ return [source]
+ else:
+ raise Exception(f"Only string and DataFrame types are allowed as a "
+ f"source, {type(source)} was provided.")
+
+ # Push data to required staging location
if uri.scheme == "gs":
- dir_path, file_name, source_path = export_dataframe_to_local(df)
+ # Staging location is a Google Cloud Storage path
upload_file_to_gcs(
- source_path, uri.hostname, str(uri.path).strip("/") + "/" + file_name
+ source_path,
+ uri.hostname,
+ str(uri.path).strip("/") + "/" + file_name
)
- if len(str(dir_path)) < 5:
- raise Exception(f"Export location {dir_path} dangerous. Stopping.")
- shutil.rmtree(dir_path)
elif uri.scheme == "file":
- dir_path, file_name, source_path = export_dataframe_to_local(df, uri.path)
+ # Staging location is a file path
+ # Used for end-to-end test
+ pass
else:
raise Exception(
- f"Staging location {staging_location_uri} does not have a valid URI. Only gs:// and file:// are supported"
+ f"Staging location {staging_location_uri} does not have a "
+ f"valid URI. Only gs:// and file:// uri scheme are supported."
)
- return staging_location_uri.rstrip("/") + "/" + file_name
+ # Clean up, remove local staging file
+ if isinstance(source, pd.DataFrame) and len(str(dir_path)) > 4:
+ shutil.rmtree(dir_path)
+
+ return [staging_location_uri.rstrip("/") + "/" + file_name]
-def export_dataframe_to_local(df: pd.DataFrame, dir_path: Optional[str] = None):
+def export_dataframe_to_local(
+ df: pd.DataFrame,
+ dir_path: Optional[str] = None
+) -> Tuple[str, str, str]:
"""
- Exports a pandas dataframe to the local filesystem
+ Exports a pandas DataFrame to the local filesystem.
Args:
- df: Pandas dataframe to save
- dir_path: (optional) Absolute directory path '/data/project/subfolder/'
+ df (pd.DataFrame):
+ Pandas DataFrame to save.
+
+ dir_path (Optional[str]):
+ Absolute directory path '/data/project/subfolder/'.
+
+ Returns:
+ Tuple[str, str, str]:
+ Tuple of directory path, file name and destination path. The
+ destination path can be obtained by concatenating the directory
+ path and file name.
"""
# Create local staging location if not provided
if dir_path is None:
dir_path = tempfile.mkdtemp()
- file_name = f'{datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p")}_{str(uuid.uuid4())[:8]}.avro'
+ file_name = _get_file_name()
dest_path = f"{dir_path}/{file_name}"
# Temporarily rename datetime column to event_timestamp. Ideally we would
# force the schema with our avro writer instead.
- df.columns = ["event_timestamp" if col == "datetime" else col for col in df.columns]
+ df.columns = [
+ "event_timestamp"
+ if col == "datetime" else col
+ for col in df.columns
+ ]
try:
# Export dataset to file in local path
@@ -74,23 +163,84 @@ def export_dataframe_to_local(df: pd.DataFrame, dir_path: Optional[str] = None):
finally:
# Revert event_timestamp column to datetime
df.columns = [
- "datetime" if col == "event_timestamp" else col for col in df.columns
+ "datetime"
+ if col == "event_timestamp" else col
+ for col in df.columns
]
return dir_path, file_name, dest_path
-def upload_file_to_gcs(local_path: str, bucket: str, remote_path: str):
+def upload_file_to_gcs(local_path: str, bucket: str, remote_path: str) -> None:
"""
- Upload a file from the local file system to Google Cloud Storage (GCS)
+ Upload a file from the local file system to Google Cloud Storage (GCS).
Args:
- local_path: Local filesystem path of file to upload
- bucket: GCS bucket to upload to
- remote_path: Path within GCS bucket to upload file to, includes file name
+ local_path (str):
+ Local filesystem path of file to upload.
+
+ bucket (str):
+ GCS bucket destination to upload to.
+
+ remote_path (str):
+ Path within GCS bucket to upload file to, includes file name.
+
+ Returns:
+ None:
+ None
"""
storage_client = storage.Client(project=None)
bucket = storage_client.get_bucket(bucket)
blob = bucket.blob(remote_path)
blob.upload_from_filename(local_path)
+
+
+def _get_files(bucket: str, uri: ParseResult) -> List[str]:
+ """
+ List all available files within a Google storage bucket that matches a wild
+ card path.
+
+ Args:
+ bucket (str):
+ Google Storage bucket to reference.
+
+ uri (urllib.parse.ParseResult):
+ Wild card uri path containing the "*" character.
+ Example:
+ * gs://feast/staging_location/*
+ * gs://feast/staging_location/file_*.avro
+
+ Returns:
+ List[str]:
+ List of all available files matching the wildcard path.
+ """
+
+ storage_client = storage.Client(project=None)
+ bucket = storage_client.get_bucket(bucket)
+ path = uri.path
+
+ if "*" in path:
+ regex = re.compile(path.replace("*", ".*?").strip("/"))
+ blob_list = bucket.list_blobs(
+ prefix=path.strip("/").split("*")[0],
+ delimiter="/"
+ )
+ # File path should not be in path (file path must be longer than path)
+ return [f"{uri.scheme}://{uri.hostname}/{file}"
+ for file in [x.name for x in blob_list]
+ if re.match(regex, file) and file not in path]
+ else:
+ raise Exception(f"{path} is not a wildcard path")
+
+
+def _get_file_name() -> str:
+ """
+ Create a random file name.
+
+ Returns:
+ str:
+ Randomised file name.
+ """
+
+ return f'{datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p")}_{str(uuid.uuid4())[:8]}.avro'
diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py
index 23ba2ecb3b4..527ab481fe0 100644
--- a/sdk/python/feast/loaders/ingest.py
+++ b/sdk/python/feast/loaders/ingest.py
@@ -1,18 +1,16 @@
import logging
-import multiprocessing
-import os
-import time
from functools import partial
-from multiprocessing import Process, Queue, Pool
-from typing import Iterable
+from multiprocessing import Pool
+from typing import Iterable, List
+
import pandas as pd
-import pyarrow as pa
+import pyarrow.parquet as pq
+from feast.constants import DATETIME_COLUMN
from feast.feature_set import FeatureSet
-from feast.type_map import convert_dict_to_proto_values
+from feast.type_map import pa_column_to_timestamp_proto_column, \
+ pa_column_to_proto_column
+from feast.types import Field_pb2 as FieldProto
from feast.types.FeatureRow_pb2 import FeatureRow
-from kafka import KafkaProducer
-from tqdm import tqdm
-from feast.constants import DATETIME_COLUMN
_logger = logging.getLogger(__name__)
@@ -21,221 +19,120 @@
FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" # type: str
FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" # type: str
BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300
-CPU_COUNT = os.cpu_count() # type: int
KAFKA_CHUNK_PRODUCTION_TIMEOUT = 120 # type: int
-def _kafka_feature_row_producer(
- feature_row_queue: Queue, row_count: int, brokers, topic, ctx: dict, pbar: tqdm
-):
+def _encode_pa_tables(
+ file: str,
+ fs: FeatureSet,
+ row_group_idx: int,
+) -> List[bytes]:
"""
- Pushes Feature Rows to Kafka. Reads rows from a queue. Function will run
- until total row_count is reached.
+ Helper function to encode a PyArrow table(s) read from parquet file(s) into
+ FeatureRows.
- Args:
- feature_row_queue: Queue containing feature rows.
- row_count: Total row count to process
- brokers: Broker to push to
- topic: Topic to push to
- ctx: Context dict used to communicate with primary process
- pbar: Progress bar object
- """
-
- # Callback for failed production to Kafka
- def on_error(e):
- # Save last exception
- ctx["last_exception"] = e
-
- # Increment error count
- if "error_count" in ctx:
- ctx["error_count"] += 1
- else:
- ctx["error_count"] = 1
-
- # Callback for succeeded production to Kafka
- def on_success(meta):
- pbar.update()
-
- producer = KafkaProducer(bootstrap_servers=brokers)
- processed_rows = 0
-
- # Loop through feature rows until all rows are processed
- while processed_rows < row_count:
- # Wait if queue is empty
- if feature_row_queue.empty():
- time.sleep(1)
- producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT)
- else:
- while not feature_row_queue.empty():
- row = feature_row_queue.get()
- if row is not None:
- # Push row to Kafka
- producer.send(topic, row.SerializeToString()).add_callback(
- on_success
- ).add_errback(on_error)
- processed_rows += 1
-
- # Force an occasional flush
- if processed_rows % 10000 == 0:
- producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT)
- del row
- pbar.refresh()
-
- # Ensure that all rows are pushed
- producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT)
-
- # Using progress bar as counter is much faster than incrementing dict
- ctx["success_count"] = pbar.n
- pbar.close()
-
-
-def _encode_pa_chunks(
- tbl: pa.lib.Table,
- fs: FeatureSet,
- max_workers: int,
- df_datetime_dtype: pd.DataFrame.dtypes,
- chunk_size: int = 5000,
-) -> Iterable[FeatureRow]:
- """
- Generator function to encode rows in PyArrow table to FeatureRows by
- breaking up the table into batches.
+ This function accepts a list of file directory pointing to many parquet
+ files. All parquet files must have the same schema.
- Each batch will have its rows spread accross a pool of workers to be
- transformed into FeatureRow objects.
+ Each parquet file will be read into as a table and encoded into FeatureRows
+ using a pool of max_workers workers.
Args:
- tbl: PyArrow table to be processed.
- fs: FeatureSet describing PyArrow table.
- max_workers: Maximum number of workers.
- df_datetime_dtype: Pandas dtype of datetime column.
- chunk_size: Maximum size of each chunk when PyArrow table is batched.
-
- Returns:
- Iterable FeatureRow object.
- """
-
- pool = Pool(max_workers)
-
- # Create a partial function with static non-iterable arguments
- func = partial(
- convert_dict_to_proto_values,
- df_datetime_dtype=df_datetime_dtype,
- feature_set=fs,
- )
-
- for batch in tbl.to_batches(max_chunksize=chunk_size):
- m_df = batch.to_pandas()
- results = pool.map_async(func, m_df.to_dict("records"))
- yield from results.get()
+ file (str):
+ File directory of all the parquet file to encode.
+ Parquet file must have more than one row group.
- pool.close()
- pool.join()
- return
+ fs (feast.feature_set.FeatureSet):
+ FeatureSet describing parquet files.
+ row_group_idx(int):
+ Row group index to read and encode into byte like FeatureRow
+ protobuf objects.
-def ingest_table_to_kafka(
- feature_set: FeatureSet,
- table: pa.lib.Table,
- max_workers: int,
- chunk_size: int = 5000,
- disable_pbar: bool = False,
- timeout: int = None,
-) -> None:
+ Returns:
+ List[bytes]:
+ List of byte encoded FeatureRows from the parquet file.
"""
- Ingest a PyArrow Table to a Kafka topic based for a Feature Set
+ pq_file = pq.ParquetFile(file)
+ # Read parquet file as a PyArrow table
+ table = pq_file.read_row_group(row_group_idx)
+
+ # Add datetime column
+ datetime_col = pa_column_to_timestamp_proto_column(
+ table.column(DATETIME_COLUMN))
+
+ # Preprocess the columns by converting all its values to Proto values
+ proto_columns = {
+ field_name: pa_column_to_proto_column(field.dtype,
+ table.column(field_name))
+ for field_name, field in fs.fields.items()
+ }
+
+ feature_set = f"{fs.name}:{fs.version}"
+
+ # List to store result
+ feature_rows = []
+
+ # Loop optimization declaration(s)
+ field = FieldProto.Field
+ proto_items = proto_columns.items()
+ append = feature_rows.append
+
+ # Iterate through the rows
+ for row_idx in range(table.num_rows):
+ feature_row = FeatureRow(event_timestamp=datetime_col[row_idx],
+ feature_set=feature_set)
+ # Loop optimization declaration
+ ext = feature_row.fields.extend
+
+ # Insert field from each column
+ for k, v in proto_items:
+ ext([field(name=k, value=v[row_idx])])
+
+ # Append FeatureRow in byte string form
+ append(feature_row.SerializeToString())
+
+ return feature_rows
+
+
+def get_feature_row_chunks(
+ file: str,
+ row_groups: List[int],
+ fs: FeatureSet,
+ max_workers: int
+) -> Iterable[List[bytes]]:
+ """
+ Iterator function to encode a PyArrow table read from a parquet file to
+ FeatureRow(s).
Args:
- feature_set: FeatureSet describing PyArrow table.
- table: PyArrow table to be processed.
- max_workers: Maximum number of workers.
- chunk_size: Maximum size of each chunk when PyArrow table is batched.
- disable_pbar: Flag to indicate if tqdm progress bar should be disabled.
- timeout: Maximum time before method times out
- """
+ file (str):
+ File directory of the parquet file. The parquet file must have more
+ than one row group.
- pbar = tqdm(unit="rows", total=table.num_rows, disable=disable_pbar)
-
- # Use a small DataFrame to validate feature set schema
- ref_df = table.to_batches(max_chunksize=100)[0].to_pandas()
- df_datetime_dtype = ref_df[DATETIME_COLUMN].dtype
-
- # Validate feature set schema
- _validate_dataframe(ref_df, feature_set)
-
- # Create queue through which encoding and production will coordinate
- row_queue = Queue()
-
- # Create a context object to send and receive information across processes
- ctx = multiprocessing.Manager().dict(
- {"success_count": 0, "error_count": 0, "last_exception": ""}
- )
-
- # Create producer to push feature rows to Kafka
- ingestion_process = Process(
- target=_kafka_feature_row_producer,
- args=(
- row_queue,
- table.num_rows,
- feature_set.get_kafka_source_brokers(),
- feature_set.get_kafka_source_topic(),
- ctx,
- pbar,
- ),
- )
-
- try:
- # Start ingestion process
- print(
- f"\n(ingest table to kafka) Ingestion started for {feature_set.name}:{feature_set.version}"
- )
- ingestion_process.start()
-
- # Iterate over chunks in the table and return feature rows
- for row in _encode_pa_chunks(
- tbl=table,
- fs=feature_set,
- max_workers=max_workers,
- chunk_size=chunk_size,
- df_datetime_dtype=df_datetime_dtype,
- ):
- # Push rows onto a queue for the production process to pick up
- row_queue.put(row)
- while row_queue.qsize() > chunk_size:
- time.sleep(0.1)
- row_queue.put(None)
- except Exception as ex:
- _logger.error(f"Exception occurred: {ex}")
- finally:
- # Wait for the Kafka production to complete
- ingestion_process.join(timeout=timeout)
- failed_message = (
- ""
- if ctx["error_count"] == 0
- else f"\nFail: {ctx['error_count']}/{table.num_rows}"
- )
+ row_groups (List[int]):
+ Specific row group indexes to be read and transformed in the parquet
+ file.
- last_exception_message = (
- ""
- if ctx["last_exception"] == ""
- else f"\nLast exception:\n{ctx['last_exception']}"
- )
- print(
- f"\nIngestion statistics:"
- f"\nSuccess: {ctx['success_count']}/{table.num_rows}"
- f"{failed_message}"
- f"{last_exception_message}"
- )
+ fs (feast.feature_set.FeatureSet):
+ FeatureSet describing parquet files.
+ max_workers (int):
+ Maximum number of workers to spawn.
-def _validate_dataframe(dataframe: pd.DataFrame, feature_set: FeatureSet):
+ Returns:
+ Iterable[List[bytes]]:
+ Iterable list of byte encoded FeatureRow(s).
"""
- Validates a Pandas dataframe based on a feature set
- Args:
- dataframe: Pandas dataframe
- feature_set: Feature Set instance
- """
+ pool = Pool(max_workers)
+ func = partial(_encode_pa_tables, file, fs)
+ for chunk in pool.imap_unordered(func, row_groups):
+ yield chunk
+ return
+
+def validate_dataframe(dataframe: pd.DataFrame, feature_set: FeatureSet):
if "datetime" not in dataframe.columns:
raise ValueError(
f'Dataframe does not contain entity "datetime" in columns {dataframe.columns}'
diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
index 7573276d74a..ca13c2573bc 100644
--- a/sdk/python/feast/type_map.py
+++ b/sdk/python/feast/type_map.py
@@ -12,12 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from datetime import datetime, timezone
+from typing import List
+
import numpy as np
import pandas as pd
-from datetime import datetime, timezone
-from feast.value_type import ValueType
+import pyarrow as pa
+from feast.constants import DATETIME_COLUMN
+from feast.types import (
+ FeatureRow_pb2 as FeatureRowProto,
+ Field_pb2 as FieldProto,
+)
from feast.types.Value_pb2 import (
Value as ProtoValue,
+ ValueType as ProtoValueType,
Int64List,
Int32List,
BoolList,
@@ -26,9 +34,9 @@
StringList,
FloatList,
)
-from feast.types import FeatureRow_pb2 as FeatureRowProto, Field_pb2 as FieldProto
+from feast.value_type import ValueType
from google.protobuf.timestamp_pb2 import Timestamp
-from feast.constants import DATETIME_COLUMN
+from pyarrow.lib import TimestampType
def python_type_to_feast_value_type(
@@ -104,9 +112,9 @@ def python_type_to_feast_value_type(
return ValueType[common_item_value_type.name + "_LIST"]
else:
raise ValueError(
- f"Value type for field {name} is {value.dtype.__str__()} "
- f"but recursion is not allowed. Array types can only be one "
- f"level deep."
+ f"Value type for field {name} is {value.dtype.__str__()} but "
+ f"recursion is not allowed. Array types can only be one level "
+ f"deep."
)
return type_map[value.dtype.__str__()]
@@ -160,7 +168,7 @@ def convert_series_to_proto_values(row: pd.Series):
def convert_dict_to_proto_values(
- row: dict, df_datetime_dtype: pd.DataFrame.dtypes, feature_set
+ row: dict, df_datetime_dtype: pd.DataFrame.dtypes, feature_set
) -> FeatureRowProto.FeatureRow:
"""
Encode a dictionary describing a feature row into a FeatureRows object.
@@ -211,12 +219,14 @@ def _pd_datetime_to_timestamp_proto(dtype, value) -> Timestamp:
# If timestamp does not contain timezone, we assume it is of local
# timezone and adjust it to UTC
local_timezone = datetime.now(timezone.utc).astimezone().tzinfo
- value = value.tz_localize(local_timezone).tz_convert("UTC").tz_localize(None)
+ value = value.tz_localize(local_timezone).tz_convert("UTC").tz_localize(
+ None)
return Timestamp(seconds=int(value.timestamp()))
if dtype.__str__() == "datetime64[ns, UTC]":
return Timestamp(seconds=int(value.timestamp()))
else:
- return Timestamp(seconds=np.datetime64(value).astype("int64") // 1000000)
+ return Timestamp(
+ seconds=np.datetime64(value).astype("int64") // 1000000)
def _type_err(item, dtype):
@@ -344,3 +354,139 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
return ProtoValue(bool_val=value)
raise Exception(f"Unsupported data type: ${str(type(value))}")
+
+def pa_to_feast_value_attr(pa_type: object):
+ """
+ Returns the equivalent Feast ValueType string for the given pa.lib type.
+
+ Args:
+ pa_type (object):
+ PyArrow type.
+
+ Returns:
+ str:
+ Feast attribute name in Feast ValueType string-ed representation.
+ """
+ # Mapping of PyArrow type to attribute name in Feast ValueType strings
+ type_map = {
+ "timestamp[ms]": "int64_val",
+ "int32": "int32_val",
+ "int64": "int64_val",
+ "double": "double_val",
+ "float": "float_val",
+ "string": "string_val",
+ "binary": "bytes_val",
+ "bool": "bool_val",
+ "list": "int32_list_val",
+ "list": "int64_list_val",
+ "list": "double_list_val",
+ "list": "float_list_val",
+ "list": "string_list_val",
+ "list": "bytes_list_val",
+ "list": "bool_list_val",
+ }
+
+ return type_map[pa_type.__str__()]
+
+
+def pa_to_value_type(pa_type: object):
+ """
+ Returns the equivalent Feast ValueType for the given pa.lib type.
+
+ Args:
+ pa_type (object):
+ PyArrow type.
+
+ Returns:
+ feast.types.Value_pb2.ValueType:
+ Feast ValueType.
+
+ """
+
+ # Mapping of PyArrow to attribute name in Feast ValueType
+ type_map = {
+ "timestamp[ms]": ProtoValueType.INT64,
+ "int32": ProtoValueType.INT32,
+ "int64": ProtoValueType.INT64,
+ "double": ProtoValueType.DOUBLE,
+ "float": ProtoValueType.FLOAT,
+ "string": ProtoValueType.STRING,
+ "binary": ProtoValueType.BYTES,
+ "bool": ProtoValueType.BOOL,
+ "list": ProtoValueType.INT32_LIST,
+ "list": ProtoValueType.INT64_LIST,
+ "list": ProtoValueType.DOUBLE_LIST,
+ "list": ProtoValueType.FLOAT_LIST,
+ "list": ProtoValueType.STRING_LIST,
+ "list": ProtoValueType.BYTES_LIST,
+ "list": ProtoValueType.BOOL_LIST,
+ }
+ return type_map[pa_type.__str__()]
+
+
+def pa_to_feast_value_type(
+ value: object
+) -> ValueType:
+ type_map = {
+ "timestamp[ms]": ValueType.INT64,
+ "int32": ValueType.INT32,
+ "int64": ValueType.INT64,
+ "double": ValueType.DOUBLE,
+ "float": ValueType.FLOAT,
+ "string": ValueType.STRING,
+ "binary": ValueType.BYTES,
+ "bool": ValueType.BOOL,
+ "list": ValueType.INT32_LIST,
+ "list": ValueType.INT64_LIST,
+ "list": ValueType.DOUBLE_LIST,
+ "list": ValueType.FLOAT_LIST,
+ "list": ValueType.STRING_LIST,
+ "list": ValueType.BYTES_LIST,
+ "list": ValueType.BOOL_LIST,
+ }
+ return type_map[value.type.__str__()]
+
+
+def pa_column_to_timestamp_proto_column(
+ column: pa.lib.ChunkedArray
+) -> Timestamp:
+ if not isinstance(column.type, TimestampType):
+ raise Exception("Only TimestampType columns are allowed")
+
+ proto_column = []
+ for val in column:
+ timestamp = Timestamp()
+ timestamp.FromMicroseconds(
+ micros=int(val.as_py().timestamp() * 1_000_000))
+ proto_column.append(timestamp)
+ return proto_column
+
+
+def pa_column_to_proto_column(
+ feast_value_type,
+ column: pa.lib.ChunkedArray
+) -> List[ProtoValue]:
+ type_map = {ValueType.INT32: "int32_val",
+ ValueType.INT64: "int64_val",
+ ValueType.FLOAT: "float_val",
+ ValueType.DOUBLE: "double_val",
+ ValueType.STRING: "string_val",
+ ValueType.BYTES: "bytes_val",
+ ValueType.BOOL: "bool_val",
+ ValueType.BOOL_LIST: {"bool_list_val": BoolList},
+ ValueType.BYTES_LIST: {"bytes_list_val": BytesList},
+ ValueType.STRING_LIST: {"string_list_val": StringList},
+ ValueType.FLOAT_LIST: {"float_list_val": FloatList},
+ ValueType.DOUBLE_LIST: {"double_list_val": DoubleList},
+ ValueType.INT32_LIST: {"int32_list_val": Int32List},
+ ValueType.INT64_LIST: {"int64_list_val": Int64List}, }
+
+ value = type_map[feast_value_type]
+ # Process list types
+ if type(value) == dict:
+ list_param_name = list(value.keys())[0]
+ return [ProtoValue(
+ **{list_param_name: value[list_param_name](val=x.as_py())})
+ for x in column]
+ else:
+ return [ProtoValue(**{value: x.as_py()}) for x in column]
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 66cad904b01..420d1a39afc 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -32,7 +32,7 @@
"googleapis-common-protos==1.*",
"google-cloud-bigquery-storage==0.7.*",
"grpcio==1.*",
- "pandas==0.*",
+ "pandas>=0.25.0",
"pandavro==1.5.*",
"protobuf>=3.10",
"PyYAML==5.1.*",
@@ -44,6 +44,7 @@
"pyarrow>=0.15.1",
"numpy",
"google",
+ "confluent_kafka"
]
# README file from Feast repo root directory
diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py
index 2243ebfd1b3..9ef6e3d56fb 100644
--- a/sdk/python/tests/test_client.py
+++ b/sdk/python/tests/test_client.py
@@ -381,7 +381,7 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker):
)
# Need to create a mock producer
- with patch("feast.loaders.ingest.KafkaProducer") as mocked_queue:
+ with patch("feast.client.get_producer") as mocked_queue:
# Ingest data into Feast
client.ingest("driver-feature-set", dataframe)
@@ -445,6 +445,6 @@ def test_feature_set_types_success(self, client, dataframe, mocker):
)
# Need to create a mock producer
- with patch("feast.loaders.ingest.KafkaProducer") as mocked_queue:
+ with patch("feast.client.get_producer") as mocked_queue:
# Ingest data into Feast
client.ingest(all_types_fs, dataframe)
diff --git a/serving/pom.xml b/serving/pom.xml
index ab9efaff26e..1e8d1b83a68 100644
--- a/serving/pom.xml
+++ b/serving/pom.xml
@@ -21,7 +21,7 @@
4.0.0
- feast
+ dev.feast
feast-parent
${revision}
@@ -47,10 +47,6 @@
false
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
-
org.apache.maven.plugins
maven-failsafe-plugin
@@ -74,6 +70,12 @@
+
+ dev.feast
+ datatypes-java
+ ${project.version}
+
+
org.slf4j
@@ -141,7 +143,19 @@
redis.clients
jedis
-
+
+
+ com.datastax.cassandra
+ cassandra-driver-core
+ 3.4.0
+
+
+ io.netty
+ *
+
+
+
+
com.google.guava
guava
@@ -227,6 +241,13 @@
spring-boot-starter-test
test
+
+
+ org.cassandraunit
+ cassandra-unit-shaded
+ 3.11.2.0
+ test
+
diff --git a/serving/sample_cassandra_config.yml b/serving/sample_cassandra_config.yml
new file mode 100644
index 00000000000..ca3d4cbbcca
--- /dev/null
+++ b/serving/sample_cassandra_config.yml
@@ -0,0 +1,13 @@
+name: serving
+type: CASSANDRA
+cassandra_config:
+ bootstrap_hosts: localhost
+ port: 9042
+ keyspace: feast
+ table_name: feature_store
+ replication_options:
+ class: SimpleStrategy
+ replication_factor: 1
+subscriptions:
+ - name: "*"
+ version: ">0"
diff --git a/serving/src/main/java/feast/serving/FeastProperties.java b/serving/src/main/java/feast/serving/FeastProperties.java
index e511835b0aa..356f808127b 100644
--- a/serving/src/main/java/feast/serving/FeastProperties.java
+++ b/serving/src/main/java/feast/serving/FeastProperties.java
@@ -85,6 +85,15 @@ public static class StoreProperties {
private String configPath;
private int redisPoolMaxSize;
private int redisPoolMaxIdle;
+ private int cassandraPoolCoreLocalConnections;
+ private int cassandraPoolMaxLocalConnections;
+ private int cassandraPoolCoreRemoteConnections;
+ private int cassandraPoolMaxRemoteConnections;
+ private int cassandraPoolMaxRequestsLocalConnection;
+ private int cassandraPoolMaxRequestsRemoteConnection;
+ private int cassandraPoolNewLocalConnectionThreshold;
+ private int cassandraPoolNewRemoteConnectionThreshold;
+ private int cassandraPoolTimeoutMillis;
public String getConfigPath() {
return this.configPath;
@@ -98,6 +107,42 @@ public int getRedisPoolMaxIdle() {
return this.redisPoolMaxIdle;
}
+ public int getCassandraPoolCoreLocalConnections() {
+ return this.cassandraPoolCoreLocalConnections;
+ }
+
+ public int getCassandraPoolMaxLocalConnections() {
+ return this.cassandraPoolMaxLocalConnections;
+ }
+
+ public int getCassandraPoolCoreRemoteConnections() {
+ return this.cassandraPoolCoreRemoteConnections;
+ }
+
+ public int getCassandraPoolMaxRemoteConnections() {
+ return this.cassandraPoolMaxRemoteConnections;
+ }
+
+ public int getCassandraPoolMaxRequestsLocalConnection() {
+ return this.cassandraPoolMaxRequestsLocalConnection;
+ }
+
+ public int getCassandraPoolMaxRequestsRemoteConnection() {
+ return this.cassandraPoolMaxRequestsRemoteConnection;
+ }
+
+ public int getCassandraPoolNewLocalConnectionThreshold() {
+ return this.cassandraPoolNewLocalConnectionThreshold;
+ }
+
+ public int getCassandraPoolNewRemoteConnectionThreshold() {
+ return this.cassandraPoolNewRemoteConnectionThreshold;
+ }
+
+ public int getCassandraPoolTimeoutMillis() {
+ return this.cassandraPoolTimeoutMillis;
+ }
+
public void setConfigPath(String configPath) {
this.configPath = configPath;
}
@@ -109,6 +154,46 @@ public void setRedisPoolMaxSize(int redisPoolMaxSize) {
public void setRedisPoolMaxIdle(int redisPoolMaxIdle) {
this.redisPoolMaxIdle = redisPoolMaxIdle;
}
+
+ public void setCassandraPoolCoreLocalConnections(int cassandraPoolCoreLocalConnections) {
+ this.cassandraPoolCoreLocalConnections = cassandraPoolCoreLocalConnections;
+ }
+
+ public void setCassandraPoolMaxLocalConnections(int cassandraPoolMaxLocalConnections) {
+ this.cassandraPoolMaxLocalConnections = cassandraPoolMaxLocalConnections;
+ }
+
+ public void setCassandraPoolCoreRemoteConnections(int cassandraPoolCoreRemoteConnections) {
+ this.cassandraPoolCoreRemoteConnections = cassandraPoolCoreRemoteConnections;
+ }
+
+ public void setCassandraPoolMaxRemoteConnections(int cassandraPoolMaxRemoteConnections) {
+ this.cassandraPoolMaxRemoteConnections = cassandraPoolMaxRemoteConnections;
+ }
+
+ public void setCassandraPoolMaxRequestsLocalConnection(
+ int cassandraPoolMaxRequestsLocalConnection) {
+ this.cassandraPoolMaxRequestsLocalConnection = cassandraPoolMaxRequestsLocalConnection;
+ }
+
+ public void setCassandraPoolMaxRequestsRemoteConnection(
+ int cassandraPoolMaxRequestsRemoteConnection) {
+ this.cassandraPoolMaxRequestsRemoteConnection = cassandraPoolMaxRequestsRemoteConnection;
+ }
+
+ public void setCassandraPoolNewLocalConnectionThreshold(
+ int cassandraPoolNewLocalConnectionThreshold) {
+ this.cassandraPoolNewLocalConnectionThreshold = cassandraPoolNewLocalConnectionThreshold;
+ }
+
+ public void setCassandraPoolNewRemoteConnectionThreshold(
+ int cassandraPoolNewRemoteConnectionThreshold) {
+ this.cassandraPoolNewRemoteConnectionThreshold = cassandraPoolNewRemoteConnectionThreshold;
+ }
+
+ public void setCassandraPoolTimeoutMillis(int cassandraPoolTimeoutMillis) {
+ this.cassandraPoolTimeoutMillis = cassandraPoolTimeoutMillis;
+ }
}
public static class JobProperties {
diff --git a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java
index 08b9655e3e1..81994941d52 100644
--- a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java
+++ b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java
@@ -16,6 +16,10 @@
*/
package feast.serving.configuration;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.HostDistance;
+import com.datastax.driver.core.PoolingOptions;
+import com.datastax.driver.core.Session;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.storage.Storage;
@@ -23,19 +27,26 @@
import feast.core.StoreProto.Store;
import feast.core.StoreProto.Store.BigQueryConfig;
import feast.core.StoreProto.Store.Builder;
+import feast.core.StoreProto.Store.CassandraConfig;
import feast.core.StoreProto.Store.RedisConfig;
import feast.core.StoreProto.Store.StoreType;
import feast.core.StoreProto.Store.Subscription;
import feast.serving.FeastProperties;
import feast.serving.FeastProperties.JobProperties;
+import feast.serving.FeastProperties.StoreProperties;
import feast.serving.service.BigQueryServingService;
import feast.serving.service.CachedSpecService;
+import feast.serving.service.CassandraServingService;
import feast.serving.service.JobService;
import feast.serving.service.NoopJobService;
import feast.serving.service.RedisServingService;
import feast.serving.service.ServingService;
import io.opentracing.Tracer;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -76,6 +87,13 @@ private Store setStoreConfig(Store.Builder builder, Map options)
.build();
return builder.setBigqueryConfig(bqConfig).build();
case CASSANDRA:
+ CassandraConfig cassandraConfig =
+ CassandraConfig.newBuilder()
+ .setBootstrapHosts(options.get("host"))
+ .setPort(Integer.parseInt(options.get("port")))
+ .setKeyspace(options.get("keyspace"))
+ .build();
+ return builder.setCassandraConfig(cassandraConfig).build();
default:
throw new IllegalArgumentException(
String.format(
@@ -135,6 +153,47 @@ public ServingService servingService(
storage);
break;
case CASSANDRA:
+ StoreProperties storeProperties = feastProperties.getStore();
+ PoolingOptions poolingOptions = new PoolingOptions();
+ poolingOptions.setCoreConnectionsPerHost(
+ HostDistance.LOCAL, storeProperties.getCassandraPoolCoreLocalConnections());
+ poolingOptions.setCoreConnectionsPerHost(
+ HostDistance.REMOTE, storeProperties.getCassandraPoolCoreRemoteConnections());
+ poolingOptions.setMaxConnectionsPerHost(
+ HostDistance.LOCAL, storeProperties.getCassandraPoolMaxLocalConnections());
+ poolingOptions.setMaxConnectionsPerHost(
+ HostDistance.REMOTE, storeProperties.getCassandraPoolMaxRemoteConnections());
+ poolingOptions.setMaxRequestsPerConnection(
+ HostDistance.LOCAL, storeProperties.getCassandraPoolMaxRequestsLocalConnection());
+ poolingOptions.setMaxRequestsPerConnection(
+ HostDistance.REMOTE, storeProperties.getCassandraPoolMaxRequestsRemoteConnection());
+ poolingOptions.setNewConnectionThreshold(
+ HostDistance.LOCAL, storeProperties.getCassandraPoolNewLocalConnectionThreshold());
+ poolingOptions.setNewConnectionThreshold(
+ HostDistance.REMOTE, storeProperties.getCassandraPoolNewRemoteConnectionThreshold());
+ poolingOptions.setPoolTimeoutMillis(storeProperties.getCassandraPoolTimeoutMillis());
+ CassandraConfig cassandraConfig = store.getCassandraConfig();
+ List contactPoints =
+ Arrays.stream(cassandraConfig.getBootstrapHosts().split(","))
+ .map(h -> new InetSocketAddress(h, cassandraConfig.getPort()))
+ .collect(Collectors.toList());
+ Cluster cluster =
+ Cluster.builder()
+ .addContactPointsWithPorts(contactPoints)
+ .withPoolingOptions(poolingOptions)
+ .build();
+ // Session in Cassandra is thread-safe and maintains connections to cluster nodes internally
+ // Recommended to use one session per keyspace instead of open and close connection for each
+ // request
+ Session session = cluster.connect();
+ servingService =
+ new CassandraServingService(
+ session,
+ cassandraConfig.getKeyspace(),
+ cassandraConfig.getTableName(),
+ specService,
+ tracer);
+ break;
case UNRECOGNIZED:
case INVALID:
throw new IllegalArgumentException(
diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java
index 701e146ee5d..7a950e3c8a9 100644
--- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java
+++ b/serving/src/main/java/feast/serving/service/BigQueryServingService.java
@@ -32,6 +32,7 @@
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.storage.Storage;
import feast.core.FeatureSetProto.FeatureSetSpec;
import feast.serving.ServingAPIProto;
@@ -56,10 +57,13 @@
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
+import org.joda.time.Duration;
import org.slf4j.Logger;
public class BigQueryServingService implements ServingService {
+ // Default no of millis for which a temporary table should exist before it is deleted in BigQuery.
+ public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.standardDays(1).getMillis();
private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryServingService.class);
private final BigQuery bigquery;
@@ -182,15 +186,15 @@ private Table loadEntities(DatasetSource datasetSource) {
switch (datasetSource.getDatasetSourceCase()) {
case FILE_SOURCE:
try {
- String tableName = generateTemporaryTableName();
- log.info("Loading entity dataset to table {}.{}.{}", projectId, datasetId, tableName);
- TableId tableId = TableId.of(projectId, datasetId, tableName);
- // Currently only avro supported
+ // Currently only AVRO format is supported
if (datasetSource.getFileSource().getDataFormat() != DataFormat.DATA_FORMAT_AVRO) {
throw Status.INVALID_ARGUMENT
- .withDescription("Invalid file format, only avro supported")
+ .withDescription("Invalid file format, only AVRO is supported.")
.asRuntimeException();
}
+
+ TableId tableId = TableId.of(projectId, datasetId, createTempTableName());
+ log.info("Loading entity rows to: {}.{}.{}", projectId, datasetId, tableId.getTable());
LoadJobConfiguration loadJobConfiguration =
LoadJobConfiguration.of(
tableId, datasetSource.getFileSource().getFileUrisList(), FormatOptions.avro());
@@ -198,6 +202,13 @@ private Table loadEntities(DatasetSource datasetSource) {
loadJobConfiguration.toBuilder().setUseAvroLogicalTypes(true).build();
Job job = bigquery.create(JobInfo.of(loadJobConfiguration));
job.waitFor();
+ TableInfo expiry =
+ bigquery
+ .getTable(tableId)
+ .toBuilder()
+ .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+ .build();
+ bigquery.update(expiry);
loadedEntityTable = bigquery.getTable(tableId);
if (!loadedEntityTable.exists()) {
throw new RuntimeException(
@@ -207,7 +218,7 @@ private Table loadEntities(DatasetSource datasetSource) {
} catch (Exception e) {
log.error("Exception has occurred in loadEntities method: ", e);
throw Status.INTERNAL
- .withDescription("Failed to load entity dataset into store")
+ .withDescription("Failed to load entity dataset into store: " + e.toString())
.withCause(e)
.asRuntimeException();
}
@@ -219,20 +230,23 @@ private Table loadEntities(DatasetSource datasetSource) {
}
}
- private String generateTemporaryTableName() {
- String source = String.format("feastserving%d", System.currentTimeMillis());
- String guid = UUID.nameUUIDFromBytes(source.getBytes()).toString();
- String suffix = guid.substring(0, Math.min(guid.length(), 10)).replaceAll("-", "");
- return String.format("temp_%s", suffix);
- }
-
private TableId generateUUIDs(Table loadedEntityTable) {
try {
String uuidQuery =
createEntityTableUUIDQuery(generateFullTableName(loadedEntityTable.getTableId()));
- QueryJobConfiguration queryJobConfig = QueryJobConfiguration.newBuilder(uuidQuery).build();
+ QueryJobConfiguration queryJobConfig =
+ QueryJobConfiguration.newBuilder(uuidQuery)
+ .setDestinationTable(TableId.of(projectId, datasetId, createTempTableName()))
+ .build();
Job queryJob = bigquery.create(JobInfo.of(queryJobConfig));
queryJob.waitFor();
+ TableInfo expiry =
+ bigquery
+ .getTable(queryJobConfig.getDestinationTable())
+ .toBuilder()
+ .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+ .build();
+ bigquery.update(expiry);
queryJobConfig = queryJob.getConfiguration();
return queryJobConfig.getDestinationTable();
} catch (InterruptedException | BigQueryException e) {
@@ -242,4 +256,8 @@ private TableId generateUUIDs(Table loadedEntityTable) {
.asRuntimeException();
}
}
+
+ public static String createTempTableName() {
+ return "_" + UUID.randomUUID().toString().replace("-", "");
+ }
}
diff --git a/serving/src/main/java/feast/serving/service/CassandraServingService.java b/serving/src/main/java/feast/serving/service/CassandraServingService.java
new file mode 100644
index 00000000000..e9a7dff8ac3
--- /dev/null
+++ b/serving/src/main/java/feast/serving/service/CassandraServingService.java
@@ -0,0 +1,153 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.querybuilder.QueryBuilder;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Timestamp;
+import feast.serving.ServingAPIProto.FeatureSetRequest;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow;
+import feast.serving.util.ValueUtil;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.FieldProto.Field;
+import feast.types.ValueProto.Value;
+import io.opentracing.Scope;
+import io.opentracing.Tracer;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class CassandraServingService extends OnlineServingService {
+
+ private final Session session;
+ private final String keyspace;
+ private final String tableName;
+ private final Tracer tracer;
+
+ public CassandraServingService(
+ Session session,
+ String keyspace,
+ String tableName,
+ CachedSpecService specService,
+ Tracer tracer) {
+ super(specService, tracer);
+ this.session = session;
+ this.keyspace = keyspace;
+ this.tableName = tableName;
+ this.tracer = tracer;
+ }
+
+ @Override
+ List createLookupKeys(
+ List featureSetEntityNames,
+ List entityRows,
+ FeatureSetRequest featureSetRequest) {
+ try (Scope scope = tracer.buildSpan("Cassandra-makeCassandraKeys").startActive(true)) {
+ String featureSetId =
+ String.format("%s:%s", featureSetRequest.getName(), featureSetRequest.getVersion());
+ return entityRows.stream()
+ .map(row -> createCassandraKey(featureSetId, featureSetEntityNames, row))
+ .collect(Collectors.toList());
+ }
+ }
+
+ @Override
+ protected boolean isEmpty(ResultSet response) {
+ return response.isExhausted();
+ }
+
+ /**
+ * Send a list of get request as an mget
+ *
+ * @param keys list of string keys
+ * @return list of {@link FeatureRow} in primitive byte representation for each key
+ */
+ @Override
+ protected List getAll(List keys) {
+ List results = new ArrayList<>();
+ for (String key : keys) {
+ results.add(
+ session.execute(
+ QueryBuilder.select()
+ .column("entities")
+ .column("feature")
+ .column("value")
+ .writeTime("value")
+ .as("writetime")
+ .from(keyspace, tableName)
+ .where(QueryBuilder.eq("entities", key))));
+ }
+ return results;
+ }
+
+ @Override
+ FeatureRow parseResponse(ResultSet resultSet) {
+ List fields = new ArrayList<>();
+ Instant instant = Instant.now();
+ while (!resultSet.isExhausted()) {
+ Row row = resultSet.one();
+ long microSeconds = row.getLong("writetime");
+ instant =
+ Instant.ofEpochSecond(
+ TimeUnit.MICROSECONDS.toSeconds(microSeconds),
+ TimeUnit.MICROSECONDS.toNanos(
+ Math.floorMod(microSeconds, TimeUnit.SECONDS.toMicros(1))));
+ try {
+ fields.add(
+ Field.newBuilder()
+ .setName(row.getString("feature"))
+ .setValue(Value.parseFrom(ByteBuffer.wrap(row.getBytes("value").array())))
+ .build());
+ } catch (InvalidProtocolBufferException e) {
+ e.printStackTrace();
+ }
+ }
+ return FeatureRow.newBuilder()
+ .addAllFields(fields)
+ .setEventTimestamp(
+ Timestamp.newBuilder()
+ .setSeconds(instant.getEpochSecond())
+ .setNanos(instant.getNano())
+ .build())
+ .build();
+ }
+
+ /**
+ * Create cassandra keys
+ *
+ * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1
+ * @param featureSetEntityNames entity names that belong to the featureSet
+ * @param entityRow entityRow to build the key from
+ * @return String
+ */
+ private static String createCassandraKey(
+ String featureSet, List featureSetEntityNames, EntityRow entityRow) {
+ Map fieldsMap = entityRow.getFieldsMap();
+ List res = new ArrayList<>();
+ for (String entityName : featureSetEntityNames) {
+ res.add(entityName + "=" + ValueUtil.toString(fieldsMap.get(entityName)));
+ }
+ return featureSet + ":" + String.join("|", res);
+ }
+}
diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java
new file mode 100644
index 00000000000..699d48c1214
--- /dev/null
+++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java
@@ -0,0 +1,236 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import static feast.serving.util.Metrics.missingKeyCount;
+import static feast.serving.util.Metrics.requestCount;
+import static feast.serving.util.Metrics.requestLatency;
+import static feast.serving.util.Metrics.staleKeyCount;
+
+import com.google.common.collect.Maps;
+import com.google.protobuf.Duration;
+import com.google.protobuf.InvalidProtocolBufferException;
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.serving.ServingAPIProto.FeastServingType;
+import feast.serving.ServingAPIProto.FeatureSetRequest;
+import feast.serving.ServingAPIProto.GetBatchFeaturesRequest;
+import feast.serving.ServingAPIProto.GetBatchFeaturesResponse;
+import feast.serving.ServingAPIProto.GetFeastServingInfoRequest;
+import feast.serving.ServingAPIProto.GetFeastServingInfoResponse;
+import feast.serving.ServingAPIProto.GetJobRequest;
+import feast.serving.ServingAPIProto.GetJobResponse;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.ValueProto.Value;
+import io.grpc.Status;
+import io.opentracing.Scope;
+import io.opentracing.Tracer;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+abstract class OnlineServingService implements ServingService {
+
+ private final CachedSpecService specService;
+ private final Tracer tracer;
+
+ OnlineServingService(CachedSpecService specService, Tracer tracer) {
+ this.specService = specService;
+ this.tracer = tracer;
+ }
+
+ @Override
+ public GetFeastServingInfoResponse getFeastServingInfo(
+ GetFeastServingInfoRequest getFeastServingInfoRequest) {
+ return GetFeastServingInfoResponse.newBuilder()
+ .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE)
+ .build();
+ }
+
+ @Override
+ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) {
+ throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException();
+ }
+
+ @Override
+ public GetJobResponse getJob(GetJobRequest getJobRequest) {
+ throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) {
+ try (Scope scope =
+ tracer.buildSpan("OnlineServingService-getOnlineFeatures").startActive(true)) {
+ long startTime = System.currentTimeMillis();
+ GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder =
+ GetOnlineFeaturesResponse.newBuilder();
+
+ List entityRows = request.getEntityRowsList();
+ Map> featureValuesMap =
+ entityRows.stream()
+ .collect(Collectors.toMap(er -> er, er -> Maps.newHashMap(er.getFieldsMap())));
+
+ List featureSetRequests = request.getFeatureSetsList();
+ for (FeatureSetRequest featureSetRequest : featureSetRequests) {
+
+ FeatureSetSpec featureSetSpec =
+ specService.getFeatureSet(featureSetRequest.getName(), featureSetRequest.getVersion());
+
+ List featureSetEntityNames =
+ featureSetSpec.getEntitiesList().stream()
+ .map(EntitySpec::getName)
+ .collect(Collectors.toList());
+
+ Duration defaultMaxAge = featureSetSpec.getMaxAge();
+ if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) {
+ featureSetRequest = featureSetRequest.toBuilder().setMaxAge(defaultMaxAge).build();
+ }
+
+ sendAndProcessMultiGet(
+ createLookupKeys(featureSetEntityNames, entityRows, featureSetRequest),
+ entityRows,
+ featureValuesMap,
+ featureSetRequest);
+ }
+ List fieldValues =
+ featureValuesMap.values().stream()
+ .map(m -> FieldValues.newBuilder().putAllFields(m).build())
+ .collect(Collectors.toList());
+ requestLatency.labels("getOnlineFeatures").observe(System.currentTimeMillis() - startTime);
+ return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build();
+ }
+ }
+
+ /**
+ * Create lookup keys for corresponding data stores
+ *
+ * @param featureSetEntityNames list of entity names
+ * @param entityRows list of {@link EntityRow}
+ * @param featureSetRequest {@link FeatureSetRequest}
+ * @return list of {@link LookupKeyType}
+ */
+ abstract List createLookupKeys(
+ List featureSetEntityNames,
+ List entityRows,
+ FeatureSetRequest featureSetRequest);
+
+ /**
+ * Checks whether the response is empty, i.e. feature does not exist in the store
+ *
+ * @param response {@link ResponseType}
+ * @return boolean
+ */
+ protected abstract boolean isEmpty(ResponseType response);
+
+ /**
+ * Send a list of get requests
+ *
+ * @param keys list of {@link LookupKeyType}
+ * @return list of {@link ResponseType}
+ */
+ protected abstract List getAll(List keys);
+
+ /**
+ * Parse response from data store to FeatureRow
+ *
+ * @param response {@link ResponseType}
+ * @return {@link FeatureRow}
+ */
+ abstract FeatureRow parseResponse(ResponseType response) throws InvalidProtocolBufferException;
+
+ private List getResponses(List keys) {
+ try (Scope scope = tracer.buildSpan("OnlineServingService-sendMultiGet").startActive(true)) {
+ long startTime = System.currentTimeMillis();
+ try {
+ return getAll(keys);
+ } catch (Exception e) {
+ throw Status.NOT_FOUND
+ .withDescription("Unable to retrieve feature from online store")
+ .withCause(e)
+ .asRuntimeException();
+ } finally {
+ requestLatency.labels("sendMultiGet").observe(System.currentTimeMillis() - startTime);
+ }
+ }
+ }
+
+ private void sendAndProcessMultiGet(
+ List keys,
+ List entityRows,
+ Map> featureValuesMap,
+ FeatureSetRequest featureSetRequest) {
+ List responses = getResponses(keys);
+
+ long startTime = System.currentTimeMillis();
+ try (Scope scope = tracer.buildSpan("OnlineServingService-processResponse").startActive(true)) {
+ String featureSetId =
+ String.format("%s:%d", featureSetRequest.getName(), featureSetRequest.getVersion());
+ Map nullValues =
+ featureSetRequest.getFeatureNamesList().stream()
+ .collect(
+ Collectors.toMap(
+ name -> featureSetId + ":" + name, name -> Value.newBuilder().build()));
+ for (int i = 0; i < responses.size(); i++) {
+ EntityRow entityRow = entityRows.get(i);
+ Map featureValues = featureValuesMap.get(entityRow);
+ try {
+ ResponseType response = responses.get(i);
+ if (isEmpty(response)) {
+ missingKeyCount.labels(featureSetRequest.getName()).inc();
+ featureValues.putAll(nullValues);
+ continue;
+ }
+
+ FeatureRow featureRow = parseResponse(response);
+ boolean stale = isStale(featureSetRequest, entityRow, featureRow);
+ if (stale) {
+ staleKeyCount.labels(featureSetRequest.getName()).inc();
+ featureValues.putAll(nullValues);
+ continue;
+ }
+
+ requestCount.labels(featureSetRequest.getName()).inc();
+ featureRow.getFieldsList().stream()
+ .filter(f -> featureSetRequest.getFeatureNamesList().contains(f.getName()))
+ .forEach(f -> featureValues.put(featureSetId + ":" + f.getName(), f.getValue()));
+ } catch (InvalidProtocolBufferException e) {
+ e.printStackTrace();
+ }
+ }
+ } finally {
+ requestLatency.labels("processResponse").observe(System.currentTimeMillis() - startTime);
+ }
+ }
+
+ private static boolean isStale(
+ FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) {
+ if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) {
+ return false;
+ }
+ long givenTimestamp = entityRow.getEntityTimestamp().getSeconds();
+ if (givenTimestamp == 0) {
+ givenTimestamp = System.currentTimeMillis() / 1000;
+ }
+ long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds();
+ return timeDifference > featureSetRequest.getMaxAge().getSeconds();
+ }
+}
diff --git a/serving/src/main/java/feast/serving/service/RedisServingService.java b/serving/src/main/java/feast/serving/service/RedisServingService.java
index 7c0d65dc42c..df97c25820f 100644
--- a/serving/src/main/java/feast/serving/service/RedisServingService.java
+++ b/serving/src/main/java/feast/serving/service/RedisServingService.java
@@ -16,29 +16,10 @@
*/
package feast.serving.service;
-import static feast.serving.util.Metrics.missingKeyCount;
-import static feast.serving.util.Metrics.requestCount;
-import static feast.serving.util.Metrics.requestLatency;
-import static feast.serving.util.Metrics.staleKeyCount;
-
-import com.google.common.collect.Maps;
import com.google.protobuf.AbstractMessageLite;
-import com.google.protobuf.Duration;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.FeatureSetProto.EntitySpec;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.serving.ServingAPIProto.FeastServingType;
import feast.serving.ServingAPIProto.FeatureSetRequest;
-import feast.serving.ServingAPIProto.GetBatchFeaturesRequest;
-import feast.serving.ServingAPIProto.GetBatchFeaturesResponse;
-import feast.serving.ServingAPIProto.GetFeastServingInfoRequest;
-import feast.serving.ServingAPIProto.GetFeastServingInfoResponse;
-import feast.serving.ServingAPIProto.GetJobRequest;
-import feast.serving.ServingAPIProto.GetJobResponse;
-import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest;
import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow;
-import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse;
-import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
import feast.storage.RedisProto.RedisKey;
import feast.types.FeatureRowProto.FeatureRow;
import feast.types.FieldProto.Field;
@@ -53,88 +34,18 @@
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
-public class RedisServingService implements ServingService {
+public class RedisServingService extends OnlineServingService {
private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisServingService.class);
private final JedisPool jedisPool;
- private final CachedSpecService specService;
private final Tracer tracer;
public RedisServingService(JedisPool jedisPool, CachedSpecService specService, Tracer tracer) {
+ super(specService, tracer);
this.jedisPool = jedisPool;
- this.specService = specService;
this.tracer = tracer;
}
- /** {@inheritDoc} */
- @Override
- public GetFeastServingInfoResponse getFeastServingInfo(
- GetFeastServingInfoRequest getFeastServingInfoRequest) {
- return GetFeastServingInfoResponse.newBuilder()
- .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE)
- .build();
- }
-
- /** {@inheritDoc} */
- @Override
- public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) {
- try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) {
- long startTime = System.currentTimeMillis();
- GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder =
- GetOnlineFeaturesResponse.newBuilder();
-
- List entityRows = request.getEntityRowsList();
- Map> featureValuesMap =
- entityRows.stream()
- .collect(Collectors.toMap(er -> er, er -> Maps.newHashMap(er.getFieldsMap())));
-
- List featureSetRequests = request.getFeatureSetsList();
- for (FeatureSetRequest featureSetRequest : featureSetRequests) {
-
- FeatureSetSpec featureSetSpec =
- specService.getFeatureSet(featureSetRequest.getName(), featureSetRequest.getVersion());
-
- List featureSetEntityNames =
- featureSetSpec.getEntitiesList().stream()
- .map(EntitySpec::getName)
- .collect(Collectors.toList());
-
- Duration defaultMaxAge = featureSetSpec.getMaxAge();
- if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) {
- featureSetRequest = featureSetRequest.toBuilder().setMaxAge(defaultMaxAge).build();
- }
-
- List redisKeys =
- getRedisKeys(featureSetEntityNames, entityRows, featureSetRequest);
-
- try {
- sendAndProcessMultiGet(redisKeys, entityRows, featureValuesMap, featureSetRequest);
- } catch (InvalidProtocolBufferException e) {
- throw Status.INTERNAL
- .withDescription("Unable to parse protobuf while retrieving feature")
- .withCause(e)
- .asRuntimeException();
- }
- }
- List fieldValues =
- featureValuesMap.values().stream()
- .map(m -> FieldValues.newBuilder().putAllFields(m).build())
- .collect(Collectors.toList());
- requestLatency.labels("getOnlineFeatures").observe(System.currentTimeMillis() - startTime);
- return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build();
- }
- }
-
- @Override
- public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) {
- throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException();
- }
-
- @Override
- public GetJobResponse getJob(GetJobRequest getJobRequest) {
- throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException();
- }
-
/**
* Build the redis keys for retrieval from the store.
*
@@ -143,7 +54,8 @@ public GetJobResponse getJob(GetJobRequest getJobRequest) {
* @param featureSetRequest details of the requested featureSet
* @return list of RedisKeys
*/
- private List getRedisKeys(
+ @Override
+ List createLookupKeys(
List featureSetEntityNames,
List entityRows,
FeatureSetRequest featureSetRequest) {
@@ -158,6 +70,33 @@ private List getRedisKeys(
}
}
+ @Override
+ protected boolean isEmpty(byte[] response) {
+ return response == null;
+ }
+
+ /**
+ * Send a list of get request as an mget
+ *
+ * @param keys list of {@link RedisKey}
+ * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey}
+ */
+ @Override
+ protected List getAll(List keys) {
+ Jedis jedis = jedisPool.getResource();
+ byte[][] binaryKeys =
+ keys.stream()
+ .map(AbstractMessageLite::toByteArray)
+ .collect(Collectors.toList())
+ .toArray(new byte[0][0]);
+ return jedis.mget(binaryKeys);
+ }
+
+ @Override
+ FeatureRow parseResponse(byte[] response) throws InvalidProtocolBufferException {
+ return FeatureRow.parseFrom(response);
+ }
+
/**
* Create {@link RedisKey}
*
@@ -170,6 +109,7 @@ private RedisKey makeRedisKey(
String featureSet, List featureSetEntityNames, EntityRow entityRow) {
RedisKey.Builder builder = RedisKey.newBuilder().setFeatureSet(featureSet);
Map fieldsMap = entityRow.getFieldsMap();
+ featureSetEntityNames.sort(String::compareTo);
for (int i = 0; i < featureSetEntityNames.size(); i++) {
String entityName = featureSetEntityNames.get(i);
@@ -187,93 +127,4 @@ private RedisKey makeRedisKey(
}
return builder.build();
}
-
- private void sendAndProcessMultiGet(
- List redisKeys,
- List entityRows,
- Map> featureValuesMap,
- FeatureSetRequest featureSetRequest)
- throws InvalidProtocolBufferException {
-
- List jedisResps = sendMultiGet(redisKeys);
- long startTime = System.currentTimeMillis();
- try (Scope scope = tracer.buildSpan("Redis-processResponse").startActive(true)) {
- String featureSetId =
- String.format("%s:%d", featureSetRequest.getName(), featureSetRequest.getVersion());
-
- Map nullValues =
- featureSetRequest.getFeatureNamesList().stream()
- .collect(
- Collectors.toMap(
- name -> featureSetId + ":" + name, name -> Value.newBuilder().build()));
-
- for (int i = 0; i < jedisResps.size(); i++) {
- EntityRow entityRow = entityRows.get(i);
- Map featureValues = featureValuesMap.get(entityRow);
-
- byte[] jedisResponse = jedisResps.get(i);
- if (jedisResponse == null) {
- missingKeyCount.labels(featureSetRequest.getName()).inc();
- featureValues.putAll(nullValues);
- continue;
- }
-
- FeatureRow featureRow = FeatureRow.parseFrom(jedisResponse);
-
- boolean stale = isStale(featureSetRequest, entityRow, featureRow);
- if (stale) {
- staleKeyCount.labels(featureSetRequest.getName()).inc();
- featureValues.putAll(nullValues);
- continue;
- }
-
- requestCount.labels(featureSetRequest.getName()).inc();
- featureRow.getFieldsList().stream()
- .filter(f -> featureSetRequest.getFeatureNamesList().contains(f.getName()))
- .forEach(f -> featureValues.put(featureSetId + ":" + f.getName(), f.getValue()));
- }
- } finally {
- requestLatency.labels("processResponse").observe(System.currentTimeMillis() - startTime);
- }
- }
-
- private boolean isStale(
- FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) {
- if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) {
- return false;
- }
- long givenTimestamp = entityRow.getEntityTimestamp().getSeconds();
- if (givenTimestamp == 0) {
- givenTimestamp = System.currentTimeMillis() / 1000;
- }
- long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds();
- return timeDifference > featureSetRequest.getMaxAge().getSeconds();
- }
-
- /**
- * Send a list of get request as an mget
- *
- * @param keys list of {@link RedisKey}
- * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey}
- */
- private List sendMultiGet(List keys) {
- try (Scope scope = tracer.buildSpan("Redis-sendMultiGet").startActive(true)) {
- long startTime = System.currentTimeMillis();
- try (Jedis jedis = jedisPool.getResource()) {
- byte[][] binaryKeys =
- keys.stream()
- .map(AbstractMessageLite::toByteArray)
- .collect(Collectors.toList())
- .toArray(new byte[0][0]);
- return jedis.mget(binaryKeys);
- } catch (Exception e) {
- throw Status.NOT_FOUND
- .withDescription("Unable to retrieve feature from Redis")
- .withCause(e)
- .asRuntimeException();
- } finally {
- requestLatency.labels("sendMultiGet").observe(System.currentTimeMillis() - startTime);
- }
- }
- }
}
diff --git a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
index 2d51547d0e7..43ac75d0b21 100644
--- a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
+++ b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
@@ -16,6 +16,8 @@
*/
package feast.serving.store.bigquery;
+import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS;
+import static feast.serving.service.BigQueryServingService.createTempTableName;
import static feast.serving.store.bigquery.QueryTemplater.createTimestampLimitQuery;
import com.google.auto.value.AutoValue;
@@ -27,6 +29,8 @@
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.TableResult;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
@@ -175,15 +179,17 @@ Job runBatchQuery(List featureSetQueries)
ExecutorCompletionService executorCompletionService =
new ExecutorCompletionService<>(executorService);
-
List featureSetInfos = new ArrayList<>();
for (int i = 0; i < featureSetQueries.size(); i++) {
QueryJobConfiguration queryJobConfig =
- QueryJobConfiguration.newBuilder(featureSetQueries.get(i)).build();
+ QueryJobConfiguration.newBuilder(featureSetQueries.get(i))
+ .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName()))
+ .build();
Job subqueryJob = bigquery().create(JobInfo.of(queryJobConfig));
executorCompletionService.submit(
SubqueryCallable.builder()
+ .setBigquery(bigquery())
.setFeatureSetInfo(featureSetInfos().get(i))
.setSubqueryJob(subqueryJob)
.build());
@@ -191,7 +197,8 @@ Job runBatchQuery(List featureSetQueries)
for (int i = 0; i < featureSetQueries.size(); i++) {
try {
- FeatureSetInfo featureSetInfo = executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS);
+ FeatureSetInfo featureSetInfo =
+ executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS);
featureSetInfos.add(featureSetInfo);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
jobService()
@@ -214,9 +221,19 @@ Job runBatchQuery(List featureSetQueries)
String joinQuery =
QueryTemplater.createJoinQuery(
featureSetInfos, entityTableColumnNames(), entityTableName());
- QueryJobConfiguration queryJobConfig = QueryJobConfiguration.newBuilder(joinQuery).build();
+ QueryJobConfiguration queryJobConfig =
+ QueryJobConfiguration.newBuilder(joinQuery)
+ .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName()))
+ .build();
queryJob = bigquery().create(JobInfo.of(queryJobConfig));
queryJob.waitFor();
+ TableInfo expiry =
+ bigquery()
+ .getTable(queryJobConfig.getDestinationTable())
+ .toBuilder()
+ .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+ .build();
+ bigquery().update(expiry);
return queryJob;
}
@@ -248,10 +265,19 @@ private FieldValueList getTimestampLimits(String entityTableName) {
QueryJobConfiguration getTimestampLimitsQuery =
QueryJobConfiguration.newBuilder(createTimestampLimitQuery(entityTableName))
.setDefaultDataset(DatasetId.of(projectId(), datasetId()))
+ .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName()))
.build();
try {
Job job = bigquery().create(JobInfo.of(getTimestampLimitsQuery));
TableResult getTimestampLimitsQueryResult = job.waitFor().getQueryResults();
+ TableInfo expiry =
+ bigquery()
+ .getTable(getTimestampLimitsQuery.getDestinationTable())
+ .toBuilder()
+ .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+ .build();
+ bigquery().update(expiry);
+
FieldValueList result = null;
for (FieldValueList fields : getTimestampLimitsQueryResult.getValues()) {
result = fields;
diff --git a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java
index 3c28194e7a3..b2a9009a749 100644
--- a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java
+++ b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java
@@ -16,13 +16,16 @@
*/
package feast.serving.store.bigquery;
+import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS;
import static feast.serving.store.bigquery.QueryTemplater.generateFullTableName;
import com.google.auto.value.AutoValue;
+import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.TableInfo;
import feast.serving.store.bigquery.model.FeatureSetInfo;
import java.util.concurrent.Callable;
@@ -33,6 +36,8 @@
@AutoValue
public abstract class SubqueryCallable implements Callable {
+ public abstract BigQuery bigquery();
+
public abstract FeatureSetInfo featureSetInfo();
public abstract Job subqueryJob();
@@ -44,6 +49,8 @@ public static Builder builder() {
@AutoValue.Builder
public abstract static class Builder {
+ public abstract Builder setBigquery(BigQuery bigquery);
+
public abstract Builder setFeatureSetInfo(FeatureSetInfo featureSetInfo);
public abstract Builder setSubqueryJob(Job subqueryJob);
@@ -57,6 +64,13 @@ public FeatureSetInfo call() throws BigQueryException, InterruptedException {
subqueryJob().waitFor();
subqueryConfig = subqueryJob().getConfiguration();
TableId destinationTable = subqueryConfig.getDestinationTable();
+ TableInfo expiry =
+ bigquery()
+ .getTable(destinationTable)
+ .toBuilder()
+ .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+ .build();
+ bigquery().update(expiry);
String fullTablePath = generateFullTableName(destinationTable);
return new FeatureSetInfo(featureSetInfo(), fullTablePath);
diff --git a/serving/src/main/java/feast/serving/util/ValueUtil.java b/serving/src/main/java/feast/serving/util/ValueUtil.java
new file mode 100644
index 00000000000..e3ede6af984
--- /dev/null
+++ b/serving/src/main/java/feast/serving/util/ValueUtil.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.util;
+
+import feast.types.ValueProto.Value;
+
+public class ValueUtil {
+
+ public static String toString(Value value) {
+ String strValue;
+ switch (value.getValCase()) {
+ case BYTES_VAL:
+ strValue = value.getBytesVal().toString();
+ break;
+ case STRING_VAL:
+ strValue = value.getStringVal();
+ break;
+ case INT32_VAL:
+ strValue = String.valueOf(value.getInt32Val());
+ break;
+ case INT64_VAL:
+ strValue = String.valueOf(value.getInt64Val());
+ break;
+ case DOUBLE_VAL:
+ strValue = String.valueOf(value.getDoubleVal());
+ break;
+ case FLOAT_VAL:
+ strValue = String.valueOf(value.getFloatVal());
+ break;
+ case BOOL_VAL:
+ strValue = String.valueOf(value.getBoolVal());
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format("toString method not supported for type %s", value.getValCase()));
+ }
+ return strValue;
+ }
+}
diff --git a/serving/src/main/proto/feast b/serving/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/serving/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/serving/src/main/proto/third_party b/serving/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/serving/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml
index 2daa83fbfb2..e18f49207b0 100644
--- a/serving/src/main/resources/application.yml
+++ b/serving/src/main/resources/application.yml
@@ -1,7 +1,7 @@
feast:
# This value is retrieved from project.version properties in pom.xml
# https://docs.spring.io/spring-boot/docs/current/reference/html/
- version: @project.version@
+# version: @project.version@
# GRPC service address for Feast Core
# Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information)
core-host: ${FEAST_CORE_HOST:localhost}
@@ -24,6 +24,24 @@ feast:
redis-pool-max-size: ${FEAST_REDIS_POOL_MAX_SIZE:128}
# If serving redis, the redis pool max idle conns
redis-pool-max-idle: ${FEAST_REDIS_POOL_MAX_IDLE:16}
+ # If serving cassandra, minimum connection for local host (one in same data center)
+ cassandra-pool-core-local-connections: ${FEAST_CASSANDRA_CORE_LOCAL_CONNECTIONS:1}
+ # If serving cassandra, maximum connection for local host (one in same data center)
+ cassandra-pool-max-local-connections: ${FEAST_CASSANDRA_MAX_LOCAL_CONNECTIONS:1}
+ # If serving cassandra, minimum connection for remote host (one in remote data center)
+ cassandra-pool-core-remote-connections: ${FEAST_CASSANDRA_CORE_REMOTE_CONNECTIONS:1}
+ # If serving cassandra, maximum connection for remote host (one in same data center)
+ cassandra-pool-max-remote-connections: ${FEAST_CASSANDRA_MAX_REMOTE_CONNECTIONS:1}
+ # If serving cassandra, maximum number of concurrent requests per local connection (one in same data center)
+ cassandra-pool-max-requests-local-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_LOCAL_CONNECTION:32768}
+ # If serving cassandra, maximum number of concurrent requests per remote connection (one in remote data center)
+ cassandra-pool-max-requests-remote-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_REMOTE_CONNECTION:2048}
+ # If serving cassandra, number of requests which trigger opening of new local connection (if it is available)
+ cassandra-pool-new-local-connection-threshold: ${FEAST_CASSANDRA_NEW_LOCAL_CONNECTION_THRESHOLD:30000}
+ # If serving cassandra, number of requests which trigger opening of new remote connection (if it is available)
+ cassandra-pool-new-remote-connection-threshold: ${FEAST_CASSANDRA_NEW_REMOTE_CONNECTION_THRESHOLD:400}
+ # If serving cassandra, number of milliseconds to wait to acquire connection (after that go to next available host in query plan)
+ cassandra-pool-timeout-millis: ${FEAST_CASSANDRA_POOL_TIMEOUT_MILLIS:0}
jobs:
# job-staging-location specifies the URI to store intermediate files for batch serving.
diff --git a/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java b/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java
new file mode 100644
index 00000000000..a1778251a3d
--- /dev/null
+++ b/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java
@@ -0,0 +1,244 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import static feast.serving.test.TestUtil.intValue;
+import static feast.serving.test.TestUtil.responseToMapList;
+import static feast.serving.test.TestUtil.strValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.querybuilder.Insert;
+import com.datastax.driver.core.querybuilder.QueryBuilder;
+import com.datastax.driver.core.utils.Bytes;
+import com.google.common.collect.Lists;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Timestamp;
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.serving.ServingAPIProto.FeatureSetRequest;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.serving.test.TestUtil.LocalCassandra;
+import feast.types.FeatureRowProto.FeatureRow;
+import feast.types.ValueProto.Value;
+import io.opentracing.Tracer;
+import io.opentracing.Tracer.SpanBuilder;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+public class CassandraServingServiceITTest {
+
+ @Mock CachedSpecService specService;
+
+ @Mock Tracer tracer;
+
+ private CassandraServingService cassandraServingService;
+ private Session session;
+
+ @BeforeClass
+ public static void startServer() throws InterruptedException, IOException, TTransportException {
+ LocalCassandra.start();
+ LocalCassandra.createKeyspaceAndTable();
+ }
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ FeatureSetSpec featureSetSpec =
+ FeatureSetSpec.newBuilder()
+ .addEntities(EntitySpec.newBuilder().setName("entity1"))
+ .addEntities(EntitySpec.newBuilder().setName("entity2"))
+ .build();
+
+ when(specService.getFeatureSet("featureSet", 1)).thenReturn(featureSetSpec);
+ when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class));
+
+ session =
+ new Cluster.Builder()
+ .addContactPoints(LocalCassandra.getHost())
+ .withPort(LocalCassandra.getPort())
+ .build()
+ .connect();
+
+ populateTable(session);
+
+ cassandraServingService =
+ new CassandraServingService(session, "test", "feature_store", specService, tracer);
+ }
+
+ private void populateTable(Session session) {
+ session.execute(
+ insertQuery(
+ "test", "feature_store", "featureSet:1:entity1=1|entity2=a", "feature1", intValue(1)));
+ session.execute(
+ insertQuery(
+ "test", "feature_store", "featureSet:1:entity1=1|entity2=a", "feature2", intValue(1)));
+ session.execute(
+ insertQuery(
+ "test", "feature_store", "featureSet:1:entity1=2|entity2=b", "feature1", intValue(1)));
+ session.execute(
+ insertQuery(
+ "test", "feature_store", "featureSet:1:entity1=2|entity2=b", "feature2", intValue(1)));
+ }
+
+ @AfterClass
+ public static void cleanUp() {
+ LocalCassandra.stop();
+ }
+
+ @Test
+ public void shouldReturnResponseWithValuesIfKeysPresent() {
+ GetOnlineFeaturesRequest request =
+ GetOnlineFeaturesRequest.newBuilder()
+ .addFeatureSets(
+ FeatureSetRequest.newBuilder()
+ .setName("featureSet")
+ .setVersion(1)
+ .addAllFeatureNames(Lists.newArrayList("feature1", "feature2"))
+ .build())
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", intValue(1))
+ .putFields("entity2", strValue("a")))
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", intValue(2))
+ .putFields("entity2", strValue("b")))
+ .build();
+
+ GetOnlineFeaturesResponse expected =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", intValue(1))
+ .putFields("entity2", strValue("a"))
+ .putFields("featureSet:1:feature1", intValue(1))
+ .putFields("featureSet:1:feature2", intValue(1)))
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", intValue(2))
+ .putFields("entity2", strValue("b"))
+ .putFields("featureSet:1:feature1", intValue(1))
+ .putFields("featureSet:1:feature2", intValue(1)))
+ .build();
+ GetOnlineFeaturesResponse actual = cassandraServingService.getOnlineFeatures(request);
+
+ assertThat(
+ responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray()));
+ }
+
+ @Test
+ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() {
+ GetOnlineFeaturesRequest request =
+ GetOnlineFeaturesRequest.newBuilder()
+ .addFeatureSets(
+ FeatureSetRequest.newBuilder()
+ .setName("featureSet")
+ .setVersion(1)
+ .addAllFeatureNames(Lists.newArrayList("feature1", "feature2"))
+ .build())
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", intValue(1))
+ .putFields("entity2", strValue("a")))
+ // Non-existing entity keys
+ .addEntityRows(
+ EntityRow.newBuilder()
+ .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100))
+ .putFields("entity1", intValue(55))
+ .putFields("entity2", strValue("ff")))
+ .build();
+
+ GetOnlineFeaturesResponse expected =
+ GetOnlineFeaturesResponse.newBuilder()
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", intValue(1))
+ .putFields("entity2", strValue("a"))
+ .putFields("featureSet:1:feature1", intValue(1))
+ .putFields("featureSet:1:feature2", intValue(1)))
+ // Missing keys will return empty values
+ .addFieldValues(
+ FieldValues.newBuilder()
+ .putFields("entity1", intValue(55))
+ .putFields("entity2", strValue("ff"))
+ .putFields("featureSet:1:feature1", Value.newBuilder().build())
+ .putFields("featureSet:1:feature2", Value.newBuilder().build()))
+ .build();
+ GetOnlineFeaturesResponse actual = cassandraServingService.getOnlineFeatures(request);
+
+ assertThat(
+ responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray()));
+ }
+
+ // This test should fail if cassandra no longer stores write time as microseconds or if we change
+ // the way we parse microseconds to com.google.protobuf.Timestamp
+ @Test
+ public void shouldInsertAndParseWriteTimestampInMicroSeconds()
+ throws InvalidProtocolBufferException {
+ session.execute(
+ "INSERT INTO test.feature_store (entities, feature, value)\n"
+ + " VALUES ('ENT1', 'FEAT1',"
+ + Bytes.toHexString(Value.newBuilder().build().toByteArray())
+ + ")\n"
+ + " USING TIMESTAMP 1574318287123456;");
+
+ ResultSet resultSet =
+ session.execute(
+ QueryBuilder.select()
+ .column("entities")
+ .column("feature")
+ .column("value")
+ .writeTime("value")
+ .as("writetime")
+ .from("test", "feature_store")
+ .where(QueryBuilder.eq("entities", "ENT1")));
+ FeatureRow featureRow = cassandraServingService.parseResponse(resultSet);
+
+ Assert.assertEquals(
+ Timestamp.newBuilder().setSeconds(1574318287).setNanos(123456000).build(),
+ featureRow.getEventTimestamp());
+ }
+
+ private Insert insertQuery(
+ String database, String table, String key, String featureName, Value value) {
+ return QueryBuilder.insertInto(database, table)
+ .value("entities", key)
+ .value("feature", featureName)
+ .value("value", ByteBuffer.wrap(value.toByteArray()));
+ }
+}
diff --git a/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java b/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java
new file mode 100644
index 00000000000..f965b14a640
--- /dev/null
+++ b/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.service;
+
+import static feast.serving.test.TestUtil.intValue;
+import static feast.serving.test.TestUtil.strValue;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import com.datastax.driver.core.Session;
+import feast.serving.ServingAPIProto.FeatureSetRequest;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow;
+import io.opentracing.Tracer;
+import io.opentracing.Tracer.SpanBuilder;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+public class CassandraServingServiceTest {
+
+ @Mock Session session;
+
+ @Mock CachedSpecService specService;
+
+ @Mock Tracer tracer;
+
+ private CassandraServingService cassandraServingService;
+
+ @Before
+ public void setUp() {
+ initMocks(this);
+
+ when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class));
+
+ cassandraServingService =
+ new CassandraServingService(session, "test", "feature_store", specService, tracer);
+ }
+
+ @Test
+ public void shouldConstructCassandraKeyCorrectly() {
+ List cassandraKeys =
+ cassandraServingService.createLookupKeys(
+ new ArrayList() {
+ {
+ add("entity1");
+ add("entity2");
+ }
+ },
+ new ArrayList() {
+ {
+ add(
+ EntityRow.newBuilder()
+ .putFields("entity1", intValue(1))
+ .putFields("entity2", strValue("a"))
+ .build());
+ add(
+ EntityRow.newBuilder()
+ .putFields("entity1", intValue(2))
+ .putFields("entity2", strValue("b"))
+ .build());
+ }
+ },
+ FeatureSetRequest.newBuilder().setName("featureSet").setVersion(1).build());
+
+ List expectedKeys =
+ new ArrayList() {
+ {
+ add("featureSet:1:entity1=1|entity2=a");
+ add("featureSet:1:entity1=2|entity2=b");
+ }
+ };
+
+ Assert.assertEquals(expectedKeys, cassandraKeys);
+ }
+
+ @Test(expected = Exception.class)
+ public void shouldThrowExceptionWhenCannotConstructCassandraKey() {
+ List cassandraKeys =
+ cassandraServingService.createLookupKeys(
+ new ArrayList() {
+ {
+ add("entity1");
+ add("entity2");
+ }
+ },
+ new ArrayList() {
+ {
+ add(EntityRow.newBuilder().putFields("entity1", intValue(1)).build());
+ add(
+ EntityRow.newBuilder()
+ .putFields("entity1", intValue(2))
+ .putFields("entity2", strValue("b"))
+ .build());
+ }
+ },
+ FeatureSetRequest.newBuilder().setName("featureSet").setVersion(1).build());
+ }
+}
diff --git a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java
index 890699db6d1..dd448fdf2b2 100644
--- a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java
+++ b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java
@@ -16,6 +16,9 @@
*/
package feast.serving.service;
+import static feast.serving.test.TestUtil.intValue;
+import static feast.serving.test.TestUtil.responseToMapList;
+import static feast.serving.test.TestUtil.strValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.mockito.Mockito.when;
@@ -39,7 +42,6 @@
import io.opentracing.Tracer;
import io.opentracing.Tracer.SpanBuilder;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.Test;
@@ -525,20 +527,6 @@ public void shouldFilterOutUndesiredRows() {
responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray()));
}
- private List> responseToMapList(GetOnlineFeaturesResponse response) {
- return response.getFieldValuesList().stream()
- .map(FieldValues::getFieldsMap)
- .collect(Collectors.toList());
- }
-
- private Value intValue(int val) {
- return Value.newBuilder().setInt64Val(val).build();
- }
-
- private Value strValue(String val) {
- return Value.newBuilder().setStringVal(val).build();
- }
-
private FeatureSetSpec getFeatureSetSpec() {
return FeatureSetSpec.newBuilder()
.addEntities(EntitySpec.newBuilder().setName("entity1"))
diff --git a/serving/src/test/java/feast/serving/test/TestUtil.java b/serving/src/test/java/feast/serving/test/TestUtil.java
new file mode 100644
index 00000000000..9c533590719
--- /dev/null
+++ b/serving/src/test/java/feast/serving/test/TestUtil.java
@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.test;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse;
+import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
+import feast.types.ValueProto.Value;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.thrift.transport.TTransportException;
+import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
+import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
+
+@SuppressWarnings("WeakerAccess")
+public class TestUtil {
+
+ public static class LocalCassandra {
+
+ public static void start() throws InterruptedException, IOException, TTransportException {
+ EmbeddedCassandraServerHelper.startEmbeddedCassandra();
+ }
+
+ public static void createKeyspaceAndTable() {
+ new ClassPathCQLDataSet("embedded-store/LoadCassandra.cql", true, true)
+ .getCQLStatements()
+ .forEach(s -> LocalCassandra.getSession().execute(s));
+ }
+
+ public static String getHost() {
+ return EmbeddedCassandraServerHelper.getHost();
+ }
+
+ public static int getPort() {
+ return EmbeddedCassandraServerHelper.getNativeTransportPort();
+ }
+
+ public static Cluster getCluster() {
+ return EmbeddedCassandraServerHelper.getCluster();
+ }
+
+ public static Session getSession() {
+ return EmbeddedCassandraServerHelper.getSession();
+ }
+
+ public static void stop() {
+ EmbeddedCassandraServerHelper.cleanEmbeddedCassandra();
+ }
+ }
+
+ public static List> responseToMapList(GetOnlineFeaturesResponse response) {
+ return response.getFieldValuesList().stream()
+ .map(FieldValues::getFieldsMap)
+ .collect(Collectors.toList());
+ }
+
+ public static Value intValue(int val) {
+ return Value.newBuilder().setInt64Val(val).build();
+ }
+
+ public static Value strValue(String val) {
+ return Value.newBuilder().setStringVal(val).build();
+ }
+}
diff --git a/serving/src/test/resources/embedded-store/LoadCassandra.cql b/serving/src/test/resources/embedded-store/LoadCassandra.cql
new file mode 100644
index 00000000000..c80da294b71
--- /dev/null
+++ b/serving/src/test/resources/embedded-store/LoadCassandra.cql
@@ -0,0 +1,8 @@
+CREATE KEYSPACE test with replication = {'class':'SimpleStrategy','replication_factor':1};
+
+CREATE TABLE test.feature_store(
+ entities text,
+ feature text,
+ value blob,
+ PRIMARY KEY (entities, feature)
+) WITH CLUSTERING ORDER BY (feature DESC);
\ No newline at end of file
diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py
index 067dd14a2fb..d7a717bed3b 100644
--- a/tests/e2e/bq-batch-retrieval.py
+++ b/tests/e2e/bq-batch-retrieval.py
@@ -2,6 +2,7 @@
import time
from datetime import datetime
from datetime import timedelta
+from urllib.parse import urlparse
import numpy as np
import pandas as pd
@@ -12,7 +13,15 @@
from feast.feature import Feature
from feast.feature_set import FeatureSet
from feast.type_map import ValueType
+from google.cloud import storage
from google.protobuf.duration_pb2 import Duration
+from pandavro import to_avro
+
+pd.set_option("display.max_columns", None)
+
+# How long we should wait for the Beam job to be ready for ingestion after running `feast apply`.
+# When using DirectRunner, 20 seconds is a reasonable time from past observations.
+WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY = 20
@pytest.fixture(scope="module")
@@ -30,6 +39,11 @@ def allow_dirty(pytestconfig):
return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False
+@pytest.fixture(scope="module")
+def gcs_path(pytestconfig):
+ return pytestconfig.getoption("gcs_path")
+
+
@pytest.fixture(scope="module")
def client(core_url, serving_url, allow_dirty):
# Get client for core and serving
@@ -44,6 +58,96 @@ def client(core_url, serving_url, allow_dirty):
return client
+def test_get_batch_features_with_file(client):
+ file_fs1 = FeatureSet(
+ "file_feature_set",
+ features=[Feature("feature_value", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
+
+ client.apply(file_fs1)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
+ file_fs1 = client.get_feature_set(name="file_feature_set", version=1)
+
+ N_ROWS = 10
+ time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
+ features_1_df = pd.DataFrame(
+ {
+ "datetime": [time_offset] * N_ROWS,
+ "entity_id": [i for i in range(N_ROWS)],
+ "feature_value": [f"{i}" for i in range(N_ROWS)],
+ }
+ )
+ client.ingest(file_fs1, features_1_df)
+
+ # Rename column (datetime -> event_timestamp)
+ features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"})
+
+ to_avro(df=features_1_df, file_path_or_buffer="file_feature_set.avro")
+
+ feature_retrieval_job = client.get_batch_features(
+ entity_rows="file://file_feature_set.avro", feature_ids=["file_feature_set:1:feature_value"]
+ )
+
+ output = feature_retrieval_job.to_dataframe()
+ print(output.head())
+
+ assert output["entity_id"].to_list() == [int(i) for i in output["file_feature_set_v1_feature_value"].to_list()]
+
+
+def test_get_batch_features_with_gs_path(client, gcs_path):
+ gcs_fs1 = FeatureSet(
+ "gcs_feature_set",
+ features=[Feature("feature_value", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
+
+ client.apply(gcs_fs1)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
+ gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1)
+
+ N_ROWS = 10
+ time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
+ features_1_df = pd.DataFrame(
+ {
+ "datetime": [time_offset] * N_ROWS,
+ "entity_id": [i for i in range(N_ROWS)],
+ "feature_value": [f"{i}" for i in range(N_ROWS)],
+ }
+ )
+ client.ingest(gcs_fs1, features_1_df)
+
+ # Rename column (datetime -> event_timestamp)
+ features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"})
+
+ # Output file to local
+ file_name = "gcs_feature_set.avro"
+ to_avro(df=features_1_df, file_path_or_buffer=file_name)
+
+ uri = urlparse(gcs_path)
+ bucket = uri.hostname
+ ts = int(time.time())
+ remote_path = str(uri.path).strip("/") + f"{ts}/{file_name}"
+
+ # Upload file to gcs
+ storage_client = storage.Client(project=None)
+ bucket = storage_client.get_bucket(bucket)
+ blob = bucket.blob(remote_path)
+ blob.upload_from_filename(file_name)
+
+ feature_retrieval_job = client.get_batch_features(
+ entity_rows=f"{gcs_path}{ts}/*",
+ feature_ids=["gcs_feature_set:1:feature_value"]
+ )
+
+ output = feature_retrieval_job.to_dataframe()
+ print(output.head())
+
+ assert output["entity_id"].to_list() == [int(i) for i in output["gcs_feature_set_v1_feature_value"].to_list()]
+
+
def test_order_by_creation_time(client):
proc_time_fs = FeatureSet(
"processing_time",
@@ -52,7 +156,7 @@ def test_order_by_creation_time(client):
max_age=Duration(seconds=100),
)
client.apply(proc_time_fs)
- time.sleep(10)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
proc_time_fs = client.get_feature_set(name="processing_time", version=1)
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
@@ -91,13 +195,17 @@ def test_additional_columns_in_entity_table(client):
max_age=Duration(seconds=100),
)
client.apply(add_cols_fs)
- time.sleep(10)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
add_cols_fs = client.get_feature_set(name="additional_columns", version=1)
N_ROWS = 10
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
features_df = pd.DataFrame(
- {"datetime": [time_offset] * N_ROWS, "entity_id": [i for i in range(N_ROWS)], "feature_value": ["abc"] * N_ROWS}
+ {
+ "datetime": [time_offset] * N_ROWS,
+ "entity_id": [i for i in range(N_ROWS)],
+ "feature_value": ["abc"] * N_ROWS,
+ }
)
client.ingest(add_cols_fs, features_df)
@@ -112,8 +220,8 @@ def test_additional_columns_in_entity_table(client):
feature_retrieval_job = client.get_batch_features(
entity_rows=entity_df, feature_ids=["additional_columns:1:feature_value"]
)
- output = feature_retrieval_job.to_dataframe()
- print(output.head())
+ output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"])
+ print(output.head(10))
assert np.allclose(output["additional_float_col"], entity_df["additional_float_col"])
assert output["additional_string_col"].to_list() == entity_df["additional_string_col"].to_list()
@@ -128,7 +236,7 @@ def test_point_in_time_correctness_join(client):
max_age=Duration(seconds=100),
)
client.apply(historical_fs)
- time.sleep(10)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
historical_fs = client.get_feature_set(name="historical", version=1)
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
@@ -174,11 +282,11 @@ def test_multiple_featureset_joins(client):
)
client.apply(fs1)
- time.sleep(10)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
fs1 = client.get_feature_set(name="feature_set_1", version=1)
client.apply(fs2)
- time.sleep(10)
+ time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY)
fs2 = client.get_feature_set(name="feature_set_2", version=1)
N_ROWS = 10
diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
index b37770a83f9..8ea472b6620 100644
--- a/tests/e2e/conftest.py
+++ b/tests/e2e/conftest.py
@@ -2,3 +2,4 @@ def pytest_addoption(parser):
parser.addoption("--core_url", action="store", default="localhost:6565")
parser.addoption("--serving_url", action="store", default="localhost:6566")
parser.addoption("--allow_dirty", action="store", default="False")
+ parser.addoption("--gcs_path", action="store", default="gs://feast-templocation-kf-feast/")
diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt
index 6b999421c04..0ba345a000f 100644
--- a/tests/e2e/requirements.txt
+++ b/tests/e2e/requirements.txt
@@ -1,6 +1,7 @@
mock==2.0.0
numpy==1.16.4
pandas==0.24.2
+pandavro==1.5.*
pytest==5.2.1
pytest-benchmark==3.2.2
pytest-mock==1.10.4