diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 7f6c3fe..c33ff8f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -42,6 +42,16 @@ jobs: integration-test: runs-on: ubuntu-latest needs: unit-test-java + services: + redis: + image: redis + ports: + - 6389:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v2 with: @@ -54,7 +64,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 348362e..417a99d 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@master with: version: '290.0.1' export_default_credentials: true diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index 2acd1cd..cf8527a 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -2,7 +2,8 @@ name: mirror on: push: - branches: master + branches: + - master tags: - 'v*.*.*' diff --git a/.gitmodules b/.gitmodules index a908f12..df8838f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "deps/feast"] path = deps/feast url = https://github.com/feast-dev/feast - branch = v0.9-branch + branch = v0.10-branch diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e4e04e8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,167 @@ +# Development Guide: feast-java +> The higher level [Development Guide](https://docs.feast.dev/contributing/development-guide) +> gives contributing to Feast codebase as a whole. + +### Overview +This guide is targeted at developers looking to contribute to Feast components in +the feast-java Repository: +- [Feast Core](#feast-core) +- [Feast Serving](#feast-serving) +- [Feast Java Client](#feast-java-client) + +> Don't see the Feast component that you want to contribute to here? +> Check out the [Development Guide](https://docs.feast.dev/contributing/development-guide) +> to learn how Feast components are distributed over multiple repositories. + +#### Common Setup +Common Environment Setup for all feast-java Feast components: +1. feast-java contains submodules that need to be updated: +```sh +git submodule init +git submodule update --recursive +``` +2. Ensure following development tools are installed: +- Java SE Development Kit 11, Maven 3.6, `make` + +#### Code Style +feast-java's codebase conforms to the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). + +Automatically format the code to conform the style guide by: + +```sh +# formats all code in the feast-java repository +mvn spotless:apply +``` + +> If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) +> if you'd like to use the IDE's reformat function. + +#### Project Makefile +The Project Makefile provides useful shorthands for common development tasks: + + +Run all Unit tests: +``` +make test-java +``` + +Run all Integration tests: +``` +make test-java-integration +``` + +Building Docker images for Feast Core & Feast Serving: +``` +make build-docker REGISTRY=gcr.io/kf-feast VERSION=develop +``` + + +#### IDE Setup +If you're using IntelliJ, some additional steps may be needed to make sure IntelliJ autocomplete works as expected. +Specifically, proto-generated code is not indexed by IntelliJ. To fix this, navigate to the following window in IntelliJ: +`Project Structure > Modules > datatypes-java`, and mark the following folders as `Source` directorys: +- target/generated-sources/protobuf/grpc-java +- target/generated-sources/protobuf/java +- target/generated-sources/annotations + + +## Feast Core +### Environment Setup +Setting up your development environment for Feast Core: +1. Complete the feast-java [Common Setup](#common-setup) +2. Boot up a PostgreSQL instance (version 11 and above). Example of doing so via Docker: +```sh +# spawn a PostgreSQL instance as a Docker container running in the background +docker run \ + --rm -it -d \ + --name postgres \ + -e POSTGRES_DB=postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=password \ + -p 5432:5432 postgres:12-alpine +``` + +### Configuration +Feast Core is configured using it's [application.yml](https://docs.feast.dev/reference/configuration-reference#1-feast-core-and-feast-online-serving). + +### Building and Running +1. Build / Compile Feast Core with Maven to produce an executable Feast Core JAR +```sh +mvn package -pl core --also-make -Dmaven.test.skip=true +``` + +2. Run Feast Core using the built JAR: +```sh +# where X.X.X is the version of the Feast Core JAR built +java -jar core/target/feast-core-X.X.X-exec.jar +``` + +### Unit / Integration Tests +Unit & Integration Tests can be used to verify functionality: +```sh +# run unit tests +mvn test -pl core --also-make +# run integration tests +mvn verify -pl core --also-make +``` + +## Feast Serving +### Environment Setup +Setting up your development environment for Feast Serving: +1. Complete the feast-java [Common Setup](#common-setup) +2. Boot up a Redis instance (version 5.x). Example of doing so via Docker: +```sh +docker run --name redis --rm -it -d -p 6379:6379 redis:5-alpine +``` + +> Feast Serving requires a running Feast Core instance to retrieve Feature metadata +> in order to serve features. See the [Feast Core section](#feast-core) for +> how to get a Feast Core instance running. + +### Configuration +Feast Serving is configured using it's [application.yml](https://docs.feast.dev/reference/configuration-reference#1-feast-core-and-feast-online-serving). + +### Building and Running +1. Build / Compile Feast Serving with Maven to produce an executable Feast Serving JAR +```sh +mvn package -pl serving --also-make -Dmaven.test.skip=true + +2. Run Feast Serving using the built JAR: +```sh +# where X.X.X is the version of the Feast serving JAR built +java -jar serving/target/feast-serving-X.X.X-exec.jar +``` + +### Unit / Integration Tests +Unit & Integration Tests can be used to verify functionality: +```sh +# run unit tests +mvn test -pl serving --also-make +# run integration tests +mvn verify -pl serving --also-make +``` + +## Feast Java Client +### Environment Setup +Setting up your development environment for Feast Java SDK: +1. Complete the feast-java [Common Setup](#common-setup) + +> Feast Java Client is a Java Client for retrieving Features from a running Feast Serving instance. +> See the [Feast Serving Section](#feast-serving) section for how to get a Feast Serving instance running. + +### Configuration +Feast Java Client is [configured as code](https://docs.feast.dev/v/master/reference/configuration-reference#4-feast-java-and-go-sdk) + +### Building +1. Build / Compile Feast Java Client with Maven: + +```sh +mvn package -pl sdk/java --also-make -Dmaven.test.skip=true +``` + +### Unit Tests +Unit Tests can be used to verify functionality: + +```sh +mvn package -pl sdk/java test --also-make +``` diff --git a/Makefile b/Makefile index b1f95dc..7d00913 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,10 @@ push-serving-docker: docker push $(REGISTRY)/feast-serving:$(VERSION) build-core-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . + docker build --no-cache --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . build-serving-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . + docker build --no-cache --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . # Versions diff --git a/README.md b/README.md index 2bcd549..98df035 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -# Feast Java components +# Feast Java components (deprecated) [![complete](https://github.com/feast-dev/feast-java/actions/workflows/complete.yml/badge.svg)](https://github.com/feast-dev/feast-java/actions/workflows/complete.yml) +### Note: This repository worked with Feast 0.9 and before. Please look at http://github.com/feast-dev/feast for the more up to date version of this repo. + ### Overview This repository contains the following Feast components. @@ -19,28 +21,11 @@ This repository contains the following Feast components. * Feast Serving has a dependency on Feast Core. * The Go and Python Clients are not a part of this repository. -### Running tests - -To run unit tests: - -``` -make test-java -``` - -To run integration tests: - -``` -make test-java-integration -``` - -### Building docker images - -In order to build development versions of the Core and Serving images, please run the following commands: - -``` -build-docker REGISTRY=gcr.io/kf-feast VERSION=develop -``` +### Contributing +Guides on Contributing: +- [Contribution Process for Feast](https://docs.feast.dev/v/master/contributing/contributing) +- [Development Guide for Feast](https://docs.feast.dev/contributing/development-guide) +- [Development Guide for feast-java (this repository)](./CONTRIBUTING.md) ### Installing using Helm - -Please see the Helm charts in [charts](infra/charts). \ No newline at end of file +Please see the Helm charts in [charts](infra/charts). diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java index f82a804..8d49b38 100644 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -115,7 +115,7 @@ public ConsumerFactory testConsumerFactory() { /** * Truncates all tables in Database (between tests or flows). Retries on deadlock * - * @throws SQLException + * @throws SQLException when a SQL exception occurs */ public static void cleanTables() throws SQLException { Connection connection = @@ -156,7 +156,12 @@ public static void cleanTables() throws SQLException { } } - /** Used to determine SequentialFlows */ + /** + * Used to determine SequentialFlows + * + * @param testInfo test info + * @return true if test is sequential + */ public Boolean isSequentialTest(TestInfo testInfo) { try { testInfo.getTestClass().get().asSubclass(SequentialFlow.class); diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 0606c75..8a0dbb0 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -53,6 +53,28 @@ public static Triple getDefaultSubscription() { return defaultSubscription; } + public static String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + public static StoreProto.Store getDefaultStore() { return defaultStore; } @@ -227,6 +249,10 @@ public static ValueProto.Value createDoubleValue(double value) { return ValueProto.Value.newBuilder().setDoubleVal(value).build(); } + public static ValueProto.Value createInt32Value(int value) { + return ValueProto.Value.newBuilder().setInt32Val(value).build(); + } + public static ValueProto.Value createInt64Value(long value) { return ValueProto.Value.newBuilder().setInt64Val(value).build(); } @@ -247,6 +273,18 @@ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityR .build(); } + public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createCompoundEntityRow( + ImmutableMap entityNameValues, long seconds) { + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.Builder entityRow = + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(seconds)); + + entityNameValues.entrySet().stream() + .forEach(entry -> entityRow.putFields(entry.getKey(), entry.getValue())); + + return entityRow.build(); + } + public static DataSource createKinesisDataSourceSpec( String region, String streamName, String classPath, String timestampColumn) { return DataSource.newBuilder() diff --git a/common-test/src/main/java/feast/common/util/TestUtil.java b/common-test/src/main/java/feast/common/util/TestUtil.java index ee355d3..49e6cc7 100644 --- a/common-test/src/main/java/feast/common/util/TestUtil.java +++ b/common-test/src/main/java/feast/common/util/TestUtil.java @@ -44,6 +44,10 @@ public static void setupAuditLogger() { /** * Compare if two Feature Table specs are equal. Disregards order of features/entities in spec. + * + * @param spec one spec + * @param otherSpec the other spec + * @return true if specs equal */ public static boolean compareFeatureTableSpec(FeatureTableSpec spec, FeatureTableSpec otherSpec) { spec = diff --git a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java b/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java index 0f42325..57aafa2 100644 --- a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java +++ b/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java @@ -45,6 +45,7 @@ public class GoogleAuthCredentials extends CallCredentials { * * @param options a map of options, Required unless specified: audience - Optional, Sets the * target audience of the token obtained. + * @throws IOException if credentials are not available */ public GoogleAuthCredentials(Map options) throws IOException { String targetAudience = options.getOrDefault("audience", "https://localhost"); diff --git a/common/src/main/java/feast/common/logging/AuditLogger.java b/common/src/main/java/feast/common/logging/AuditLogger.java index 0b9901e..5f70fbf 100644 --- a/common/src/main/java/feast/common/logging/AuditLogger.java +++ b/common/src/main/java/feast/common/logging/AuditLogger.java @@ -65,6 +65,7 @@ public AuditLogger(LoggingProperties loggingProperties, BuildProperties buildPro /** * Log the handling of a Protobuf message by a service call. * + * @param level log level * @param entryBuilder with all fields set except instance. */ public static void logMessage(Level level, MessageAuditLogEntry.Builder entryBuilder) { diff --git a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java index cec85b7..4fdeaee 100644 --- a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java @@ -21,10 +21,10 @@ /** ActionAuditLogEntry records an action being taken on a specific resource */ @AutoValue public abstract class ActionAuditLogEntry extends AuditLogEntry { - /** The name of the action taken on the resource. */ + /** @return The name of the action taken on the resource. */ public abstract String getAction(); - /** The target resource of which the action was taken on. */ + /** @return The target resource of which the action was taken on. */ public abstract LogResource getResource(); /** @@ -34,6 +34,7 @@ public abstract class ActionAuditLogEntry extends AuditLogEntry { * @param version The version of Feast producing this {@link AuditLogEntry}. * @param resource The target resource of which the action was taken on. * @param action The name of the action being taken on the given resource. + * @return log entry that records an action being taken on a specific resource */ public static ActionAuditLogEntry of( String component, String version, LogResource resource, String action) { diff --git a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java index 9aa8fcb..8148c47 100644 --- a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java @@ -29,15 +29,27 @@ public abstract class AuditLogEntry { public final String application = "Feast"; - /** The name of the Feast component producing this {@link AuditLogEntry} */ + /** + * The name of the Feast component producing this {@link AuditLogEntry} + * + * @return the component + */ public abstract String getComponent(); - /** The version of Feast producing this {@link AuditLogEntry} */ + /** + * The version of Feast producing this {@link AuditLogEntry} + * + * @return version + */ public abstract String getVersion(); public abstract AuditLogEntryKind getKind(); - /** Return a structured JSON representation of this {@link AuditLogEntry} */ + /** + * Return a structured JSON representation of this {@link AuditLogEntry} + * + * @return structured JSON representation + */ public String toJSON() { Gson gson = new Gson(); return gson.toJson(this); diff --git a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java index 745cc12..6e5072f 100644 --- a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java @@ -34,32 +34,35 @@ /** MessageAuditLogEntry records the handling of a Protobuf message by a service call. */ @AutoValue public abstract class MessageAuditLogEntry extends AuditLogEntry { - /** Id used to identify the service call that the log entry is recording */ + /** @return Id used to identify the service call that the log entry is recording */ public abstract UUID getId(); - /** The name of the service that was used to handle the service call. */ + /** @return The name of the service that was used to handle the service call. */ public abstract String getService(); - /** The name of the method that was used to handle the service call. */ + /** @return The name of the method that was used to handle the service call. */ public abstract String getMethod(); - /** The request Protobuf {@link Message} that was passed to the Service in the service call. */ + /** + * @return The request Protobuf {@link Message} that was passed to the Service in the service + * call. + */ public abstract Message getRequest(); /** - * The response Protobuf {@link Message} that was passed to the Service in the service call. May - * be an {@link Empty} protobuf no request could be collected due to an error. + * @return The response Protobuf {@link Message} that was passed to the Service in the service + * call. May be an {@link Empty} protobuf no request could be collected due to an error. */ public abstract Message getResponse(); /** - * The authenticated identity that was assumed during the handling of the service call. For - * example, the user id or email that identifies the user making the call. Empty if the service - * call is not authenticated. + * @return The authenticated identity that was assumed during the handling of the service call. + * For example, the user id or email that identifies the user making the call. Empty if the + * service call is not authenticated. */ public abstract String getIdentity(); - /** The result status code of the service call. */ + /** @return The result status code of the service call. */ public abstract Code getStatusCode(); @AutoValue.Builder diff --git a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java index 0f139b7..224f10e 100644 --- a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java @@ -21,10 +21,10 @@ /** TransitionAuditLogEntry records a transition in state/status in a specific resource. */ @AutoValue public abstract class TransitionAuditLogEntry extends AuditLogEntry { - /** The resource which the state/status transition occured. */ + /** @return The resource which the state/status transition occured. */ public abstract LogResource getResource(); - /** The end status with the resource transition to. */ + /** @return The end status with the resource transition to. */ public abstract String getStatus(); /** @@ -35,6 +35,7 @@ public abstract class TransitionAuditLogEntry extends AuditLogEntry { * @param version The version of Feast producing this {@link AuditLogEntry}. * @param resource the resource which the transtion occured * @param status the end status which the resource transitioned to. + * @return log entry to record a transition in state/status in a specific resource */ public static TransitionAuditLogEntry of( String component, String version, LogResource resource, String status) { diff --git a/common/src/main/java/feast/common/models/FeatureTable.java b/common/src/main/java/feast/common/models/FeatureTable.java index d4712e7..88fac15 100644 --- a/common/src/main/java/feast/common/models/FeatureTable.java +++ b/common/src/main/java/feast/common/models/FeatureTable.java @@ -25,6 +25,7 @@ public class FeatureTable { * Accepts FeatureTableSpec object and returns its reference in String * "project/featuretable_name". * + * @param project project name * @param featureTableSpec {@link FeatureTableSpec} * @return String format of FeatureTableReference */ @@ -36,6 +37,7 @@ public static String getFeatureTableStringRef(String project, FeatureTableSpec f * Accepts FeatureReferenceV2 object and returns its reference in String * "project/featuretable_name". * + * @param project project name * @param featureReference {@link FeatureReferenceV2} * @return String format of FeatureTableReference */ diff --git a/common/src/main/java/feast/common/models/FeatureV2.java b/common/src/main/java/feast/common/models/FeatureV2.java index 8debca3..8420cca 100644 --- a/common/src/main/java/feast/common/models/FeatureV2.java +++ b/common/src/main/java/feast/common/models/FeatureV2.java @@ -34,4 +34,17 @@ public static String getFeatureStringRef(FeatureReferenceV2 featureReference) { } return ref; } + + /** + * Accepts either a feature reference of the form "featuretable_name:feature_name" or just a + * feature name, and returns just the feature name. For example, given either + * "driver_hourly_stats:conv_rate" or "conv_rate", "conv_rate" would be returned. + * + * @param featureReference {String} + * @return Base feature name of the feature reference + */ + public static String getFeatureName(String featureReference) { + String[] tokens = featureReference.split(":", 2); + return tokens[tokens.length - 1]; + } } diff --git a/common/src/main/java/feast/common/validators/OneOfStringValidator.java b/common/src/main/java/feast/common/validators/OneOfStringValidator.java index 42428bd..924953a 100644 --- a/common/src/main/java/feast/common/validators/OneOfStringValidator.java +++ b/common/src/main/java/feast/common/validators/OneOfStringValidator.java @@ -29,7 +29,7 @@ public class OneOfStringValidator implements ConstraintValidator[] groups() default {}; - /** An attribute payload that can be used to assign custom payload objects to a constraint. */ + /** + * @return An attribute payload that can be used to assign custom payload objects to a constraint. + */ Class[] payload() default {}; /** @return Default value that is returned if no allowed values are configured */ diff --git a/core/src/main/java/feast/core/config/WebSecurityConfig.java b/core/src/main/java/feast/core/config/WebSecurityConfig.java index 0f48111..5c66730 100644 --- a/core/src/main/java/feast/core/config/WebSecurityConfig.java +++ b/core/src/main/java/feast/core/config/WebSecurityConfig.java @@ -43,7 +43,7 @@ public WebSecurityConfig(FeastProperties feastProperties) { * Allows for custom web security rules to be applied. * * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception + * @throws Exception unexpected exception */ @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java index 67477da..2bfe20f 100644 --- a/core/src/main/java/feast/core/model/DataSource.java +++ b/core/src/main/java/feast/core/model/DataSource.java @@ -87,6 +87,7 @@ public DataSource(SourceType type) { * @param spec Protobuf representation of DataSource to construct from. * @throws IllegalArgumentException when provided with a invalid Protobuf spec * @throws UnsupportedOperationException if source type is unsupported. + * @return data source */ public static DataSource fromProto(DataSourceProto.DataSource spec) { DataSource source = new DataSource(spec.getType()); @@ -132,7 +133,11 @@ public static DataSource fromProto(DataSourceProto.DataSource spec) { return source; } - /** Convert this DataSource to its Protobuf representation. */ + /** + * Convert this DataSource to its Protobuf representation. + * + * @return protobuf representation + */ public DataSourceProto.DataSource toProto() { DataSourceProto.DataSource.Builder spec = DataSourceProto.DataSource.newBuilder(); spec.setType(getType()); diff --git a/core/src/main/java/feast/core/model/EntityV2.java b/core/src/main/java/feast/core/model/EntityV2.java index aeb6728..d72bef1 100644 --- a/core/src/main/java/feast/core/model/EntityV2.java +++ b/core/src/main/java/feast/core/model/EntityV2.java @@ -65,6 +65,11 @@ public EntityV2() { * *

This data model supports Scalar Entity and would allow ease of discovery of entities and * reasoning when used in association with FeatureTable. + * + * @param name name + * @param description description + * @param type type + * @param labels labels */ public EntityV2( String name, String description, ValueType.Enum type, Map labels) { diff --git a/core/src/main/java/feast/core/model/FeatureTable.java b/core/src/main/java/feast/core/model/FeatureTable.java index 479c11e..9b9bdef 100644 --- a/core/src/main/java/feast/core/model/FeatureTable.java +++ b/core/src/main/java/feast/core/model/FeatureTable.java @@ -155,7 +155,9 @@ public static FeatureTable fromProto( /** * Update the FeatureTable from the given Protobuf representation. * + * @param projectName project name * @param spec the Protobuf spec to update the FeatureTable from. + * @param entityRepo repository * @throws IllegalArgumentException if the update will make prohibited changes. */ public void updateFromProto( @@ -211,7 +213,11 @@ public void updateFromProto( this.revision++; } - /** Convert this Feature Table to its Protobuf representation */ + /** + * Convert this Feature Table to its Protobuf representation + * + * @return protobuf representation + */ public FeatureTableProto.FeatureTable toProto() { // Convert field types to Protobuf compatible types Timestamp creationTime = TypeConversion.convertTimestamp(getCreated()); @@ -319,6 +325,7 @@ private Map getFeaturesRefToFeaturesMap(List featu /** * Returns a list of Features if FeatureTable's Feature contains all labels in labelsFilter * + * @param features features * @param labelsFilter contain labels that should be attached to FeatureTable's features * @return List of Features */ diff --git a/core/src/main/java/feast/core/model/FeatureV2.java b/core/src/main/java/feast/core/model/FeatureV2.java index f25e951..d0a6082 100644 --- a/core/src/main/java/feast/core/model/FeatureV2.java +++ b/core/src/main/java/feast/core/model/FeatureV2.java @@ -75,7 +75,11 @@ public static FeatureV2 fromProto(FeatureTable table, FeatureSpecV2 spec) { return new FeatureV2(table, spec.getName(), spec.getValueType(), labelsJSON); } - /** Convert this Feature to its Protobuf representation. */ + /** + * Convert this Feature to its Protobuf representation. + * + * @return protobuf representation + */ public FeatureSpecV2 toProto() { Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); return FeatureSpecV2.newBuilder() diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 4a35d3e..ff45dcd 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -17,7 +17,7 @@ package feast.core.service; import static feast.core.validators.Matchers.checkValidCharacters; -import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk; +import static feast.core.validators.Matchers.checkValidCharactersAllowDash; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.dao.EntityRepository; @@ -105,7 +105,7 @@ public GetEntityResponse getEntity(GetEntityRequest request) { projectName = Project.DEFAULT_NAME; } - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(entityName, "entity"); EntityV2 entity = entityRepository.findEntityByNameAndProject_Name(entityName, projectName); @@ -143,13 +143,13 @@ public ListFeaturesResponse listFeatures(ListFeaturesRequest.Filter filter) { List entities = filter.getEntitiesList(); Map labels = filter.getLabelsMap(); - checkValidCharactersAllowAsterisk(project, "project"); - // Autofill default project if project not specified if (project.isEmpty()) { project = Project.DEFAULT_NAME; } + checkValidCharactersAllowDash(project, "project"); + // Currently defaults to all FeatureTables List featureTables = tableRepository.findAllByProject_Name(project); @@ -200,7 +200,7 @@ public ListEntitiesResponse listEntities(ListEntitiesRequest.Filter filter) { project = Project.DEFAULT_NAME; } - checkValidCharacters(project, "project"); + checkValidCharactersAllowDash(project, "project"); List entities = entityRepository.findAllByProject_Name(project); @@ -262,6 +262,7 @@ public ListStoresResponse listStores(ListStoresRequest.Filter filter) { * * @param newEntitySpec EntitySpecV2 that will be used to create or update an Entity. * @param projectName Project namespace of Entity which is to be created/updated + * @return response of the operation */ @Transactional public ApplyEntityResponse applyEntity( @@ -271,6 +272,8 @@ public ApplyEntityResponse applyEntity( projectName = Project.DEFAULT_NAME; } + checkValidCharactersAllowDash(projectName, "project"); + // Validate incoming entity EntityValidator.validateSpec(newEntitySpec); @@ -312,6 +315,7 @@ public ApplyEntityResponse applyEntity( * Resolves the project name by returning name if given, autofilling default project otherwise. * * @param projectName name of the project to resolve. + * @return project name */ public static String resolveProjectName(String projectName) { return (projectName.isEmpty()) ? Project.DEFAULT_NAME : projectName; @@ -322,6 +326,7 @@ public static String resolveProjectName(String projectName) { * * @param updateStoreRequest containing the new store definition * @return UpdateStoreResponse containing the new store definition + * @throws InvalidProtocolBufferException if protobuf exception occurs */ @Transactional public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) @@ -368,6 +373,8 @@ public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) public ApplyFeatureTableResponse applyFeatureTable(ApplyFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); + checkValidCharactersAllowDash(projectName, "project"); + // Check that specification provided is valid FeatureTableSpec applySpec = request.getTableSpec(); FeatureTableValidator.validateSpec(applySpec); @@ -411,7 +418,7 @@ public ListFeatureTablesResponse listFeatureTables(ListFeatureTablesRequest.Filt String projectName = resolveProjectName(filter.getProject()); Map labelsFilter = filter.getLabelsMap(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); List matchingTables = tableRepository.findAllByProject_Name(projectName); @@ -444,7 +451,7 @@ public GetFeatureTableResponse getFeatureTable(GetFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); String featureTableName = request.getName(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(featureTableName, "featureTable"); Optional retrieveTable = @@ -474,7 +481,7 @@ public void deleteFeatureTable(DeleteFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); String featureTableName = request.getName(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(featureTableName, "featureTable"); Optional existingTable = diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index bbdfa94..d2a2d0a 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -79,7 +79,7 @@ public static Map convertJsonStringToEnumMap(String jsonString) { /** * Marshals a given map into its corresponding json string * - * @param map + * @param map map to be converted * @return json string corresponding to given map */ public static String convertMapToJsonString(Map map) { @@ -89,7 +89,7 @@ public static String convertMapToJsonString(Map map) { /** * Marshals a given Enum map into its corresponding json string * - * @param map + * @param map map to be converted * @return json string corresponding to given Enum map */ public static String convertEnumMapToJsonString(Map map) { diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java index f36e360..548d18f 100644 --- a/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ b/core/src/main/java/feast/core/validators/DataSourceValidator.java @@ -24,7 +24,11 @@ import feast.proto.core.DataSourceProto.DataSource; public class DataSourceValidator { - /** Validate if the given DataSource protobuf spec is valid. */ + /** + * Validate if the given DataSource protobuf spec is valid. + * + * @param spec spec to be validated + */ public static void validate(DataSource spec) { switch (spec.getType()) { case BATCH_FILE: diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java index 5f7ddd2..b6d7fcc 100644 --- a/core/src/main/java/feast/core/validators/Matchers.java +++ b/core/src/main/java/feast/core/validators/Matchers.java @@ -29,8 +29,8 @@ public class Matchers { private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); - private static Pattern VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD = - Pattern.compile("^[a-zA-Z0-9\\-_*]*$"); + private static Pattern VALID_CHARACTERS_REGEX_WITH_DASH = + Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_-]*$"); private static String ERROR_MESSAGE_TEMPLATE = "invalid value for %s resource, %s: %s"; @@ -70,15 +70,15 @@ public static void checkValidCharacters(String input, String resource) } } - public static void checkValidCharactersAllowAsterisk(String input, String resource) + public static void checkValidCharactersAllowDash(String input, String resource) throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD.matcher(input).matches()) { + if (!VALID_CHARACTERS_REGEX_WITH_DASH.matcher(input).matches()) { throw new IllegalArgumentException( String.format( ERROR_MESSAGE_TEMPLATE, resource, input, - "argument must only contain alphanumeric characters, dashes, underscores, or an asterisk.")); + "argument must only contain alphanumeric characters, dashes, or underscores.")); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java deleted file mode 100644 index 41faee7..0000000 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=http", - }) -public class CoreServiceAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java deleted file mode 100644 index 0a09cce..0000000 --- a/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=keto", - "feast.security.authorization.options.action=actions:any", - "feast.security.authorization.options.subjectPrefix=users:", - "feast.security.authorization.options.resourcePrefix=resources:projects:", - }) -public class CoreServiceKetoAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoExternalUrl); - registry.add("feast.security.authorization.options.flavor", () -> DEFAULT_FLAVOR); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/core/src/test/java/feast/core/logging/CoreLoggingIT.java deleted file mode 100644 index 0f137b4..0000000 --- a/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Streams; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.logging.entry.AuditLogEntryKind; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Status.Code; -import io.grpc.StatusRuntimeException; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest( - properties = { - "feast.logging.audit.enabled=true", - "feast.logging.audit.messageLogging.enabled=true", - "feast.logging.audit.messageLogging.destination=console" - }) -public class CoreLoggingIT extends BaseIT { - private static TestLogAppender testAuditLogAppender; - private static CoreServiceBlockingStub coreService; - private static CoreServiceFutureStub asyncCoreService; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int coreGrpcPort) - throws InterruptedException, ExecutionException { - LoggerContext logContext = (LoggerContext) LogManager.getContext(false); - // NOTE: As log appender state is shared across tests use a different method - // for each test and filter by method name to ensure that you only get logs - // for a specific test. - testAuditLogAppender = logContext.getConfiguration().getAppender("TestAuditLogAppender"); - - // Connect to core service. - Channel channel = - ManagedChannelBuilder.forAddress("localhost", coreGrpcPort).usePlaintext().build(); - coreService = CoreServiceGrpc.newBlockingStub(channel); - asyncCoreService = CoreServiceGrpc.newFutureStub(channel); - - // Preflight a request to core service stubs to verify connection - coreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()); - asyncCoreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()).get(); - } - - /** Check that messsage audit log are produced on service call */ - @Test - public void shouldProduceMessageAuditLogsOnCall() - throws InterruptedException, InvalidProtocolBufferException { - // Generate artifical load on feast core. - UpdateStoreRequest request = - UpdateStoreRequest.newBuilder().setStore(DataGenerator.getDefaultStore()).build(); - UpdateStoreResponse response = coreService.updateStore(request); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Check message audit logs are produced for each audit log. - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "UpdateStore"); - assertEquals(1, logJsonObjects.size()); - JsonObject logObj = logJsonObjects.get(0); - - // Extract & Check that request/response are returned correctly - String requestJson = logObj.getAsJsonObject("request").toString(); - UpdateStoreRequest.Builder gotRequest = UpdateStoreRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - UpdateStoreResponse.Builder gotResponse = UpdateStoreResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - } - - /** Check that message audit logs are produced when server encounters an error */ - @Test - public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { - // Send a bad request which should cause Core to error - ListFeatureTablesRequest request = - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) - .build(); - - boolean hasExpectedException = false; - Code statusCode = null; - try { - coreService.listFeatureTables(request); - } catch (StatusRuntimeException e) { - hasExpectedException = true; - statusCode = e.getStatus().getCode(); - } - assertTrue(hasExpectedException); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); - - assertEquals(1, logJsonObjects.size()); - JsonObject logJsonObject = logJsonObjects.get(0); - // Check correct status code is tracked on error. - assertEquals(logJsonObject.get("statusCode").getAsString(), statusCode.toString()); - } - - /** Check that expected message audit logs are produced when under load. */ - @Test - public void shouldProduceExpectedAuditLogsUnderLoad() - throws InterruptedException, ExecutionException { - // Generate artifical requests on core to simulate load. - int LOAD_SIZE = 40; // Total number of requests to send. - int BURST_SIZE = 5; // Number of requests to send at once. - - ListStoresRequest request = ListStoresRequest.getDefaultInstance(); - List responses = new LinkedList<>(); - for (int i = 0; i < LOAD_SIZE; i += 5) { - List> futures = new LinkedList<>(); - for (int j = 0; j < BURST_SIZE; j++) { - futures.add(asyncCoreService.listStores(request)); - } - - responses.addAll(Futures.allAsList(futures).get()); - } - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - - // Pull message audit logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListStores"); - assertEquals(responses.size(), logJsonObjects.size()); - - // Extract & Check that request/response are returned correctly - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - Streams.zip( - responses.stream(), - logJsonObjects.stream(), - (response, logObj) -> Pair.of(response, logObj)) - .forEach( - responseLogJsonPair -> { - ListStoresResponse response = responseLogJsonPair.getLeft(); - JsonObject logObj = responseLogJsonPair.getRight(); - - ListStoresRequest.Builder gotRequest = null; - ListStoresResponse.Builder gotResponse = null; - try { - String requestJson = logObj.getAsJsonObject("request").toString(); - gotRequest = ListStoresRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - gotResponse = ListStoresResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - }); - } - - /** - * Filter and Parse out Message Audit Logs from the given logsStrings for the given method name - */ - private List parseMessageJsonLogObjects(List logsStrings, String methodName) { - JsonParser jsonParser = new JsonParser(); - // copy to prevent concurrent modification. - return logsStrings.stream() - .map(logJSON -> jsonParser.parse(logJSON).getAsJsonObject()) - // Filter to only include message audit logs - .filter( - logObj -> - logObj - .getAsJsonPrimitive("kind") - .getAsString() - .equals(AuditLogEntryKind.MESSAGE.toString()) - // filter by method name to ensure logs from other tests do not interfere with - // test - && logObj.get("method").getAsString().equals(methodName)) - .collect(Collectors.toList()); - } -} diff --git a/core/src/test/java/feast/core/service/ProjectServiceTest.java b/core/src/test/java/feast/core/service/ProjectServiceTest.java index e09580f..afff1e5 100644 --- a/core/src/test/java/feast/core/service/ProjectServiceTest.java +++ b/core/src/test/java/feast/core/service/ProjectServiceTest.java @@ -16,10 +16,8 @@ */ package feast.core.service; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; import feast.core.dao.ProjectRepository; @@ -29,16 +27,12 @@ import java.util.Optional; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.mockito.Mock; public class ProjectServiceTest { @Mock private ProjectRepository projectRepository; - @Rule public final ExpectedException expectedException = ExpectedException.none(); - private ProjectService projectService; @Before @@ -75,8 +69,9 @@ public void shouldArchiveProjectIfItExists() { @Test public void shouldNotArchiveDefaultProject() { - expectedException.expect(IllegalArgumentException.class); - this.projectService.archiveProject(Project.DEFAULT_NAME); + assertThrows( + IllegalArgumentException.class, + () -> this.projectService.archiveProject(Project.DEFAULT_NAME)); } @Test(expected = IllegalArgumentException.class) diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 8851d87..d6d0f74 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -170,7 +170,7 @@ public void shouldThrowExceptionGivenWildcardProject() { equalTo( String.format( "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", + + "argument must only contain alphanumeric characters, dashes, or underscores.", filter.getProject()))); } } @@ -226,7 +226,7 @@ public void shouldThrowExceptionGivenWildcardProject() { equalTo( String.format( "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", + + "argument must only contain alphanumeric characters, dashes, or underscores.", filter.getProject()))); } } diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java index c44bf50..ba965a7 100644 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ b/core/src/test/java/feast/core/util/TypeConversionTest.java @@ -17,8 +17,8 @@ package feast.core.util; import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.*; import com.google.protobuf.Timestamp; import java.util.*; diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java index 1733212..559330a 100644 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ b/core/src/test/java/feast/core/validators/MatchersTest.java @@ -19,14 +19,12 @@ import static feast.core.validators.Matchers.checkLowerSnakeCase; import static feast.core.validators.Matchers.checkUpperSnakeCase; import static feast.core.validators.Matchers.checkValidClassPath; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.base.Strings; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; public class MatchersTest { - @Rule public final ExpectedException exception = ExpectedException.none(); @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { @@ -42,15 +40,15 @@ public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { @Test public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( + String in = "redis"; + assertThrows( + IllegalArgumentException.class, + () -> checkUpperSnakeCase(in, "featuretable"), Strings.lenientFormat( "invalid value for %s resource, %s: %s", "featuretable", "redis", "argument must be in upper snake case, and cannot include any special characters.")); - String in = "redis"; - checkUpperSnakeCase(in, "featuretable"); } @Test @@ -61,15 +59,15 @@ public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() { @Test public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( + String in = "Invalid_feature name"; + assertThrows( + IllegalArgumentException.class, + () -> checkLowerSnakeCase(in, "feature"), Strings.lenientFormat( "invalid value for %s resource, %s: %s", "feature", "Invalid_feature name", "argument must be in lower snake case, and cannot include any special characters.")); - String in = "Invalid_feature name"; - checkLowerSnakeCase(in, "feature"); } @Test @@ -80,13 +78,11 @@ public void checkValidClassPathSuccess() { @Test public void checkValidClassPathEmpty() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("", "FeatureTable"); + assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("", "FeatureTable")); } @Test public void checkValidClassPathDigits() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("123", "FeatureTable"); + assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("123", "FeatureTable")); } } diff --git a/datatypes/java/README.md b/datatypes/java/README.md index 2759f82..7fc355f 100644 --- a/datatypes/java/README.md +++ b/datatypes/java/README.md @@ -16,7 +16,7 @@ Dependency Coordinates dev.feast datatypes-java - 0.26.0-SNAPSHOT + 0.26.2 ``` diff --git a/deps/feast b/deps/feast index 8010d2f..2541c91 160000 --- a/deps/feast +++ b/deps/feast @@ -1 +1 @@ -Subproject commit 8010d2f35d3f876db54a31b8012b13009cd5eba2 +Subproject commit 2541c91c9238ef09ffd74e45e74116a31d7f2daa diff --git a/infra/charts/feast-core/Chart.yaml b/infra/charts/feast-core/Chart.yaml index 7065b52..ac52a32 100644 --- a/infra/charts/feast-core/Chart.yaml +++ b/infra/charts/feast-core/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Core: Feature registry for Feast." name: feast-core -version: 0.26.0 -appVersion: 0.26.0 +version: 0.26.2 +appVersion: 0.26.2 keywords: - machine learning - big data diff --git a/infra/charts/feast-core/README.md b/infra/charts/feast-core/README.md index d6bed1b..bae6786 100644 --- a/infra/charts/feast-core/README.md +++ b/infra/charts/feast-core/README.md @@ -2,7 +2,7 @@ feast-core ========== Feast Core: Feature registry for Feast. -Current chart version is `0.26.0` +Current chart version is `0.26.2` Source code can be found [here](https://github.com/feast-dev/feast-java) diff --git a/infra/charts/feast-serving/Chart.yaml b/infra/charts/feast-serving/Chart.yaml index 9ecab9c..e9a4c41 100644 --- a/infra/charts/feast-serving/Chart.yaml +++ b/infra/charts/feast-serving/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Serving: Online feature serving service for Feast" name: feast-serving -version: 0.26.0 -appVersion: 0.26.0 +version: 0.26.2 +appVersion: 0.26.2 keywords: - machine learning - big data diff --git a/infra/charts/feast-serving/README.md b/infra/charts/feast-serving/README.md index 4aea435..66a982c 100644 --- a/infra/charts/feast-serving/README.md +++ b/infra/charts/feast-serving/README.md @@ -2,7 +2,7 @@ feast-serving ============= Feast Serving: Online feature serving service for Feast -Current chart version is `0.26.0` +Current chart version is `0.26.2` Source code can be found [here](https://github.com/feast-dev/feast-java) diff --git a/pom.xml b/pom.xml index 1cd7187..50d9b95 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ - 0.26.0-SNAPSHOT + 0.26.2 https://github.com/feast-dev/feast UTF-8 @@ -269,7 +269,38 @@ ${grpc.version} test - + + + + org.apache.arrow + arrow-java-root + 5.0.0 + pom + + + + + org.apache.arrow + arrow-vector + 5.0.0 + + + + + org.apache.arrow + arrow-memory + 5.0.0 + pom + + + + + org.apache.arrow + arrow-memory-netty + 5.0.0 + runtime + + io.swagger @@ -543,6 +574,9 @@ + + feast.proto.*:io.grpc.*:org.tensorflow.* + com.diffplug.spotless @@ -691,7 +725,7 @@ - ${groupId}:${artifactId} + ${project.groupId}:${project.artifactId} ${project.build.outputDirectory} diff --git a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java b/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java index bd3b34b..94c779c 100644 --- a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java +++ b/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java @@ -26,15 +26,23 @@ public abstract class SecurityConfig { /** * Enables authentication If specified, the call credentials used to provide credentials to * authenticate with Feast. + * + * @return credentials */ public abstract Optional getCredentials(); - /** Whether to use TLS transport security is use when connecting to Feast. */ + /** + * Whether to use TLS transport security is use when connecting to Feast. + * + * @return true if enabled + */ public abstract boolean isTLSEnabled(); /** * If specified and TLS is enabled, provides path to TLS certificate use the verify Service * identity. + * + * @return certificate path */ public abstract Optional getCertificatePath(); diff --git a/serving/README.md b/serving/README.md index ab530bb..cce8c7d 100644 --- a/serving/README.md +++ b/serving/README.md @@ -88,3 +88,14 @@ with open("/tmp/000000000000.avro", "rb") as f: print(df.head(5)) EOF ``` +#### Working with Feast 0.10+ +Feast serving supports reading feature values materialized into Redis by feast 0.10+. To configure this, feast-serving +needs to be able to read the registry file for the project. +The location of the registry file can be specified in the `application.yml` like so: +```yaml +feast: + registry: "src/test/resources/docker-compose/feast10/registry.db" +``` + +This changes the behaviour of feast-serving to look up feature view definitions and specifications from the registry file instead +of the core service. \ No newline at end of file diff --git a/serving/pom.xml b/serving/pom.xml index b8f675d..981c23c 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -84,12 +84,32 @@ ${project.version} + + dev.feast + feast-storage-connector-bigtable + ${project.version} + + + + dev.feast + feast-storage-connector-cassandra + ${project.version} + + dev.feast feast-common ${project.version} - + + + com.google.cloud + google-cloud-bigtable-emulator + 0.130.2 + test + + + org.slf4j @@ -129,6 +149,14 @@ spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-test + 2.3.1.RELEASE + test + + io.grpc @@ -234,6 +262,44 @@ test + + + org.apache.avro + avro + 1.10.2 + + + + + org.apache.arrow + arrow-java-root + 5.0.0 + pom + + + + + org.apache.arrow + arrow-vector + 5.0.0 + + + + + org.apache.arrow + arrow-memory + 5.0.0 + pom + + + + + org.apache.arrow + arrow-memory-netty + 5.0.0 + runtime + + com.fasterxml.jackson.dataformat @@ -307,6 +373,12 @@ 1.15.1 test + + org.testcontainers + gcloud + 1.15.2 + test + org.awaitility awaitility diff --git a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java index 2bc9743..cdf791c 100644 --- a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java +++ b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java @@ -18,11 +18,13 @@ import java.util.concurrent.ScheduledExecutorService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; import org.springframework.stereotype.Component; @Component +@ConditionalOnBean(CoreCondition.class) public class ContextClosedHandler implements ApplicationListener { @Autowired ScheduledExecutorService executor; diff --git a/serving/src/main/java/feast/serving/config/CoreCondition.java b/serving/src/main/java/feast/serving/config/CoreCondition.java new file mode 100644 index 0000000..10dabfa --- /dev/null +++ b/serving/src/main/java/feast/serving/config/CoreCondition.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from Core service. + */ +public class CoreCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") == null; + } +} diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 3b8548a..82db07c 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -26,6 +26,8 @@ import feast.common.auth.config.SecurityProperties.AuthorizationProperties; import feast.common.auth.credentials.CoreAuthenticationProperties; import feast.common.logging.config.LoggingProperties; +import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig; import feast.storage.connectors.redis.retriever.RedisStoreConfig; import io.lettuce.core.ReadFrom; @@ -70,6 +72,26 @@ public FeastProperties() {} /* Feast Core port to connect to. */ @Positive private int coreGrpcPort; + private String registry; + + public String getRegistry() { + return registry; + } + + public void setRegistry(final String registry) { + this.registry = registry; + } + + private String transformationServiceEndpoint; + + public String getTransformationServiceEndpoint() { + return transformationServiceEndpoint; + } + + public void setTransformationServiceEndpoint(final String transformationServiceEndpoint) { + this.transformationServiceEndpoint = transformationServiceEndpoint; + } + private CoreAuthenticationProperties coreAuthentication; public CoreAuthenticationProperties getCoreAuthentication() { @@ -80,7 +102,6 @@ public void setCoreAuthentication(CoreAuthenticationProperties coreAuthenticatio this.coreAuthentication = coreAuthentication; } - /* Feast Core port to connect to. */ @Positive private int coreCacheRefreshInterval; private SecurityProperties security; @@ -269,7 +290,7 @@ public void setName(String name) { } /** - * Gets the store type. Example are REDIS or REDIS_CLUSTER + * Gets the store type. Example are REDIS, REDIS_CLUSTER, BIGTABLE or CASSANDRA * * @return the store type as a String. */ @@ -301,14 +322,31 @@ public RedisClusterStoreConfig getRedisClusterConfig() { return new RedisClusterStoreConfig( this.config.get("connection_string"), ReadFrom.valueOf(this.config.get("read_from")), - Duration.parse(this.config.get("timeout"))); + Duration.parse(this.config.get("timeout")), + Boolean.valueOf(this.config.getOrDefault("ssl", "false")), + this.config.getOrDefault("password", "")); } public RedisStoreConfig getRedisConfig() { return new RedisStoreConfig( this.config.get("host"), Integer.valueOf(this.config.get("port")), - Boolean.valueOf(this.config.getOrDefault("ssl", "false"))); + Boolean.valueOf(this.config.getOrDefault("ssl", "false")), + this.config.getOrDefault("password", "")); + } + + public BigTableStoreConfig getBigtableConfig() { + return new BigTableStoreConfig( + this.config.get("project_id"), + this.config.get("instance_id"), + this.config.get("app_profile_id")); + } + + public CassandraStoreConfig getCassandraConfig() { + return new CassandraStoreConfig( + this.config.get("connection_string"), + this.config.get("data_center"), + this.config.get("keyspace")); } /** @@ -323,6 +361,8 @@ public void setConfig(Map config) { } public enum StoreType { + BIGTABLE, + CASSANDRA, REDIS, REDIS_CLUSTER; } @@ -354,7 +394,11 @@ public LoggingProperties getLogging() { return logging; } - /** Sets logging properties @@param logging the logging properties */ + /** + * Sets logging properties + * + * @param logging the logging properties + */ public void setLogging(LoggingProperties logging) { this.logging = logging; } diff --git a/serving/src/main/java/feast/serving/config/RegistryCondition.java b/serving/src/main/java/feast/serving/config/RegistryCondition.java new file mode 100644 index 0000000..621d124 --- /dev/null +++ b/serving/src/main/java/feast/serving/config/RegistryCondition.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from the Registry object. This is needed for versions of the feature store written by feast + * 0.10+. + */ +public class RegistryCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") != null; + } +} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 518f3a1..ce2aabf 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -16,43 +16,169 @@ */ package feast.serving.config; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.protobuf.AbstractMessageLite; +import feast.serving.registry.LocalRegistryRepo; import feast.serving.service.OnlineServingServiceV2; +import feast.serving.service.OnlineTransformationService; import feast.serving.service.ServingServiceV2; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; +import feast.serving.specs.FeatureSpecRetriever; +import feast.serving.specs.RegistryFeatureSpecRetriever; import feast.storage.api.retriever.OnlineRetrieverV2; +import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever; +import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraOnlineRetriever; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.*; import io.opentracing.Tracer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; @Configuration public class ServingServiceConfigV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); + @Autowired private ApplicationContext context; + + @Bean + @Lazy(true) + public BigtableDataClient bigtableClient(FeastProperties feastProperties) throws IOException { + BigTableStoreConfig config = feastProperties.getActiveStore().getBigtableConfig(); + String projectId = config.getProjectId(); + String instanceId = config.getInstanceId(); + + return BigtableDataClient.create( + BigtableDataSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setAppProfileId(config.getAppProfileId()) + .build()); + } + @Bean + @Conditional(CoreCondition.class) public ServingServiceV2 servingServiceV2( - FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) - throws InvalidProtocolBufferException, JsonProcessingException { - ServingServiceV2 servingService = null; - FeastProperties.Store store = feastProperties.getActiveStore(); + FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) { + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); + + OnlineRetrieverV2 retrieverV2; + switch (store.getType()) { + case REDIS_CLUSTER: + RedisClientAdapter redisClusterClient = + RedisClusterClient.create(store.getRedisClusterConfig()); + retrieverV2 = new OnlineRetriever(redisClusterClient, (AbstractMessageLite::toByteArray)); + break; + case REDIS: + RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); + retrieverV2 = new OnlineRetriever(redisClient, (AbstractMessageLite::toByteArray)); + break; + case BIGTABLE: + BigtableDataClient bigtableClient = context.getBean(BigtableDataClient.class); + retrieverV2 = new BigTableOnlineRetriever(bigtableClient); + break; + case CASSANDRA: + CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); + String connectionString = config.getConnectionString(); + String dataCenter = config.getDataCenter(); + String keySpace = config.getKeySpace(); + List contactPoints = + Arrays.stream(connectionString.split(",")) + .map(String::trim) + .map(cs -> cs.split(":")) + .map( + hostPort -> { + int port = hostPort.length > 1 ? Integer.parseInt(hostPort[1]) : 9042; + return new InetSocketAddress(hostPort[0], port); + }) + .collect(Collectors.toList()); + + CqlSession session = + new CqlSessionBuilder() + .addContactPoints(contactPoints) + .withLocalDatacenter(dataCenter) + .withKeyspace(keySpace) + .build(); + retrieverV2 = new CassandraOnlineRetriever(session); + break; + default: + throw new RuntimeException( + String.format("Unable to identify online store type: %s", store.getType())); + } + + final FeatureSpecRetriever featureSpecRetriever; + log.info("Created CoreFeatureSpecRetriever"); + featureSpecRetriever = new CoreFeatureSpecRetriever(specService); + + final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint(); + final OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); + + servingService = + new OnlineServingServiceV2( + retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); + + return servingService; + } + + @Bean + @Conditional(RegistryCondition.class) + public ServingServiceV2 registryBasedServingServiceV2( + FeastProperties feastProperties, Tracer tracer) { + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); + + OnlineRetrieverV2 retrieverV2; + // TODO: Support more store types, and potentially use a plugin model here. switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = RedisClusterClient.create(store.getRedisClusterConfig()); - OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient); - servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer); + retrieverV2 = new OnlineRetriever(redisClusterClient, new EntityKeySerializerV2()); break; case REDIS: RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); - OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient); - servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer); + log.info("Created EntityKeySerializerV2"); + retrieverV2 = new OnlineRetriever(redisClient, new EntityKeySerializerV2()); break; + default: + throw new RuntimeException( + String.format( + "Unable to identify online store type: %s for Regsitry Backed Serving Service", + store.getType())); } + final FeatureSpecRetriever featureSpecRetriever; + log.info("Created RegistryFeatureSpecRetriever"); + log.info("Working Directory = " + System.getProperty("user.dir")); + final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); + featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); + + final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint(); + final OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); + + servingService = + new OnlineServingServiceV2( + retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); + return servingService; } } diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java index 369d543..29d3bf0 100644 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java @@ -16,8 +16,6 @@ */ package feast.serving.config; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; import io.grpc.CallCredentials; @@ -28,15 +26,16 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; @Configuration public class SpecServiceConfig { private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); - private String feastCoreHost; - private int feastCorePort; - private int feastCachedSpecServiceRefreshInterval; + private final String feastCoreHost; + private final int feastCorePort; + private final int feastCachedSpecServiceRefreshInterval; @Autowired public SpecServiceConfig(FeastProperties feastProperties) { @@ -46,6 +45,7 @@ public SpecServiceConfig(FeastProperties feastProperties) { } @Bean + @Conditional(CoreCondition.class) public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( CachedSpecService cachedSpecStorage) { ScheduledExecutorService scheduledExecutorService = @@ -60,8 +60,8 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( } @Bean - public CachedSpecService specService(ObjectProvider callCredentials) - throws InvalidProtocolBufferException, JsonProcessingException { + @Conditional(CoreCondition.class) + public CachedSpecService specService(ObjectProvider callCredentials) { CoreSpecService coreService = new CoreSpecService(feastCoreHost, feastCorePort, callCredentials); CachedSpecService cachedSpecStorage = new CachedSpecService(coreService); diff --git a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java b/serving/src/main/java/feast/serving/config/WebSecurityConfig.java index f7b24a7..04d3f4b 100644 --- a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java +++ b/serving/src/main/java/feast/serving/config/WebSecurityConfig.java @@ -33,7 +33,7 @@ public class WebSecurityConfig extends WebSecurityConfigurerAdapter { * Allows for custom web security rules to be applied. * * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception + * @throws Exception exception */ @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java index 4bee981..ef675d4 100644 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ b/serving/src/main/java/feast/serving/controller/HealthServiceController.java @@ -19,7 +19,6 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; import io.grpc.health.v1.HealthGrpc.HealthImplBase; import io.grpc.health.v1.HealthProto.HealthCheckRequest; import io.grpc.health.v1.HealthProto.HealthCheckResponse; @@ -32,12 +31,10 @@ @GrpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class HealthServiceController extends HealthImplBase { - private CachedSpecService specService; - private ServingServiceV2 servingService; + private final ServingServiceV2 servingService; @Autowired - public HealthServiceController(CachedSpecService specService, ServingServiceV2 servingService) { - this.specService = specService; + public HealthServiceController(final ServingServiceV2 servingService) { this.servingService = servingService; } @@ -47,7 +44,7 @@ public void check( // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g. // if it's online service check that it the service can retrieve dummy/random // feature table. - // Implement similary for batch service. + // Implement similarly for batch service. try { servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance()); diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 531be39..81bbfd0 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -25,6 +25,7 @@ import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.serving.config.FeastProperties; import feast.serving.exception.SpecRetrievalException; +import feast.serving.interceptors.GrpcMonitoringContext; import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingServiceV2; import feast.serving.util.RequestHelper; @@ -86,6 +87,9 @@ public void getOnlineFeaturesV2( // project set at root level overrides the project set at feature table level this.authorizationService.authorizeRequest( SecurityContextHolder.getContext(), request.getProject()); + + // update monitoring context + GrpcMonitoringContext.getInstance().setProject(request.getProject()); } RequestHelper.validateOnlineRequest(request); Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java new file mode 100644 index 0000000..48d8d76 --- /dev/null +++ b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.interceptors; + +import java.util.Optional; + +public class GrpcMonitoringContext { + private static GrpcMonitoringContext INSTANCE; + + final ThreadLocal project = new ThreadLocal(); + + private GrpcMonitoringContext() {} + + public static GrpcMonitoringContext getInstance() { + if (INSTANCE == null) { + INSTANCE = new GrpcMonitoringContext(); + } + + return INSTANCE; + } + + public void setProject(String name) { + this.project.set(name); + } + + public Optional getProject() { + return Optional.ofNullable(this.project.get()); + } + + public void clearProject() { + this.project.set(null); + } +} diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java index bc7ed89..735f8c5 100644 --- a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java +++ b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java @@ -24,6 +24,7 @@ import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.Status; +import java.util.Optional; /** * GrpcMonitoringInterceptor intercepts GRPC calls to provide request latency histogram metrics in @@ -39,12 +40,16 @@ public Listener interceptCall( String fullMethodName = call.getMethodDescriptor().getFullMethodName(); String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); + GrpcMonitoringContext.getInstance().clearProject(); + return next.startCall( new SimpleForwardingServerCall(call) { @Override public void close(Status status, Metadata trailers) { + Optional projectName = GrpcMonitoringContext.getInstance().getProject(); + Metrics.requestLatency - .labels(methodName) + .labels(methodName, projectName.orElse("")) .observe((System.currentTimeMillis() - startCallMillis) / 1000f); Metrics.grpcRequestCount.labels(methodName, status.getCode().name()).inc(); super.close(status, trailers); diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java new file mode 100644 index 0000000..4178541 --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class LocalRegistryRepo implements RegistryRepository { + private final RegistryProto.Registry registry; + private Map featureViewNameToSpec; + private Map + onDemandFeatureViewNameToSpec; + + public LocalRegistryRepo(Path localRegistryPath) { + if (!localRegistryPath.toFile().exists()) { + throw new RuntimeException( + String.format("Local registry not found at path %s", localRegistryPath)); + } + try { + final byte[] registryContents = Files.readAllBytes(localRegistryPath); + this.registry = RegistryProto.Registry.parseFrom(registryContents); + } catch (final Exception e) { + throw new RuntimeException(e); + } + + final RegistryProto.Registry registry = this.getRegistry(); + List featureViewSpecs = + registry.getFeatureViewsList().stream() + .map(fv -> fv.getSpec()) + .collect(Collectors.toList()); + featureViewNameToSpec = + featureViewSpecs.stream() + .collect( + Collectors.toMap(FeatureViewProto.FeatureViewSpec::getName, Function.identity())); + List onDemandFeatureViewSpecs = + registry.getOnDemandFeatureViewsList().stream() + .map(odfv -> odfv.getSpec()) + .collect(Collectors.toList()); + onDemandFeatureViewNameToSpec = + onDemandFeatureViewSpecs.stream() + .collect( + Collectors.toMap( + OnDemandFeatureViewProto.OnDemandFeatureViewSpec::getName, + Function.identity())); + } + + @Override + public RegistryProto.Registry getRegistry() { + return this.registry; + } + + @Override + public FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + String featureViewName = featureReference.getFeatureTable(); + if (featureViewNameToSpec.containsKey(featureViewName)) { + return featureViewNameToSpec.get(featureViewName); + } + throw new SpecRetrievalException( + String.format("Unable to find feature view with name: %s", featureViewName)); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final FeatureViewProto.FeatureViewSpec spec = + this.getFeatureViewSpec(projectName, featureReference); + for (final FeatureProto.FeatureSpecV2 featureSpec : spec.getFeaturesList()) { + if (featureSpec.getName().equals(featureReference.getName())) { + return featureSpec; + } + } + + throw new SpecRetrievalException( + String.format( + "Unable to find feature with name: %s in feature view: %s", + featureReference.getName(), featureReference.getFeatureTable())); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + if (onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName)) { + return onDemandFeatureViewNameToSpec.get(onDemandFeatureViewName); + } + throw new SpecRetrievalException( + String.format( + "Unable to find on demand feature view with name: %s", onDemandFeatureViewName)); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + return onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName); + } +} diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java new file mode 100644 index 0000000..21a2183 --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/RegistryRepository.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; + +/** + * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry + * object. This approach is needed for a feature store created using feast 0.10+. + */ +public interface RegistryRepository { + RegistryProto.Registry getRegistry(); + + FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 70dd6f7..2cd810c 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -26,54 +26,47 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; import feast.proto.types.ValueProto; import feast.serving.exception.SpecRetrievalException; -import feast.serving.specs.CachedSpecService; +import feast.serving.specs.FeatureSpecRetriever; import feast.serving.util.Metrics; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.OnlineRetrieverV2; import io.grpc.Status; import io.opentracing.Span; import io.opentracing.Tracer; -import java.util.*; +import java.io.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; public class OnlineServingServiceV2 implements ServingServiceV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class); - private final CachedSpecService specService; private final Tracer tracer; private final OnlineRetrieverV2 retriever; - - private static final HashMap - TYPE_TO_VAL_CASE = - new HashMap<>() { - { - put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); - put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); - put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); - put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); - put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); - put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); - put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); - put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); - put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); - put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); - put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); - put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); - put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); - put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); - } - }; + private final FeatureSpecRetriever featureSpecRetriever; + private final OnlineTransformationService onlineTransformationService; public OnlineServingServiceV2( - OnlineRetrieverV2 retriever, CachedSpecService specService, Tracer tracer) { + OnlineRetrieverV2 retriever, + Tracer tracer, + FeatureSpecRetriever featureSpecRetriever, + OnlineTransformationService onlineTransformationService) { this.retriever = retriever; - this.specService = specService; this.tracer = tracer; + this.featureSpecRetriever = featureSpecRetriever; + this.onlineTransformationService = onlineTransformationService; } /** {@inheritDoc} */ @@ -87,41 +80,62 @@ public GetFeastServingInfoResponse getFeastServingInfo( @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 request) { - String projectName = request.getProject(); - List featureReferences = request.getFeaturesList(); - // Autofill default project if project is not specified + String projectName = request.getProject(); if (projectName.isEmpty()) { projectName = "default"; } - List entityRows = request.getEntityRowsList(); + // Split all feature references into non-ODFV (e.g. batch and stream) references and ODFV. + List allFeatureReferences = request.getFeaturesList(); + List featureReferences = + allFeatureReferences.stream() + .filter(r -> !this.featureSpecRetriever.isOnDemandFeatureReference(r)) + .collect(Collectors.toList()); + List onDemandFeatureReferences = + allFeatureReferences.stream() + .filter(r -> this.featureSpecRetriever.isOnDemandFeatureReference(r)) + .collect(Collectors.toList()); + + // Get the set of request data feature names and feature inputs from the ODFV references. + Pair, List> pair = + this.onlineTransformationService.extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + onDemandFeatureReferences, projectName); + Set requestDataFeatureNames = pair.getLeft(); + List onDemandFeatureInputs = pair.getRight(); + + // Add on demand feature inputs to list of feature references to retrieve. + Set addedFeatureReferences = new HashSet(); + for (FeatureReferenceV2 onDemandFeatureInput : onDemandFeatureInputs) { + if (!featureReferences.contains(onDemandFeatureInput)) { + featureReferences.add(onDemandFeatureInput); + addedFeatureReferences.add(onDemandFeatureInput); + } + } + + // Separate entity rows into entity data and request feature data. + Pair, Map>> + entityRowsAndRequestDataFeatures = + this.onlineTransformationService.separateEntityRows(requestDataFeatureNames, request); + List entityRows = + entityRowsAndRequestDataFeatures.getLeft(); + Map> requestDataFeatures = + entityRowsAndRequestDataFeatures.getRight(); + // TODO: error checking on lengths of lists in entityRows and requestDataFeatures + + // Extract values and statuses to be used later in constructing FieldValues for the response. + // The online features retrieved will augment these two data structures. List> values = entityRows.stream().map(r -> new HashMap<>(r.getFieldsMap())).collect(Collectors.toList()); List> statuses = entityRows.stream() - .map(r -> getMetadataMap(r.getFieldsMap(), false, false)) + .map( + r -> + r.getFieldsMap().entrySet().stream() + .map(entry -> Pair.of(entry.getKey(), getMetadata(entry.getValue(), false))) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight))) .collect(Collectors.toList()); - Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.setTag("entities", entityRows.size()); - storageRetrievalSpan.setTag("features", featureReferences.size()); - } - List> entityRowsFeatures = - retriever.getOnlineFeatures(projectName, entityRows, featureReferences); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.finish(); - } - - if (entityRowsFeatures.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } - String finalProjectName = projectName; Map featureMaxAges = featureReferences.stream() @@ -129,7 +143,12 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re .collect( Collectors.toMap( Function.identity(), - ref -> specService.getFeatureTableSpec(finalProjectName, ref).getMaxAge())); + ref -> this.featureSpecRetriever.getMaxAge(finalProjectName, ref))); + List entityNames = + featureReferences.stream() + .map(ref -> this.featureSpecRetriever.getEntitiesList(finalProjectName, ref)) + .findFirst() + .get(); Map featureValueTypes = featureReferences.stream() @@ -139,12 +158,33 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re Function.identity(), ref -> { try { - return specService.getFeatureSpec(finalProjectName, ref).getValueType(); + return this.featureSpecRetriever + .getFeatureSpec(finalProjectName, ref) + .getValueType(); } catch (SpecRetrievalException e) { return ValueProto.ValueType.Enum.INVALID; } })); + Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.setTag("entities", entityRows.size()); + storageRetrievalSpan.setTag("features", featureReferences.size()); + } + List> entityRowsFeatures = + retriever.getOnlineFeatures(projectName, entityRows, featureReferences, entityNames); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.finish(); + } + + if (entityRowsFeatures.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); + } + Span postProcessingSpan = tracer.buildSpan("postProcessing").start(); for (int i = 0; i < entityRows.size(); i++) { @@ -161,44 +201,35 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re if (featureReferenceFeatureMap.containsKey(featureReference)) { Feature feature = featureReferenceFeatureMap.get(featureReference); - ValueProto.Value.ValCase valueCase = feature.getFeatureValue().getValCase(); + ValueProto.Value value = + feature.getFeatureValue(featureValueTypes.get(feature.getFeatureReference())); - boolean isMatchingFeatureSpec = - checkSameFeatureSpec(featureValueTypes.get(feature.getFeatureReference()), valueCase); - boolean isOutsideMaxAge = + Boolean isOutsideMaxAge = checkOutsideMaxAge( feature, entityRow, featureMaxAges.get(feature.getFeatureReference())); - Map valueMap = - unpackValueMap(feature, isOutsideMaxAge, isMatchingFeatureSpec); - rowValues.putAll(valueMap); - - // Generate metadata for feature values and merge into entityFieldsMap - Map statusMap = - getMetadataMap(valueMap, !isMatchingFeatureSpec, isOutsideMaxAge); - rowStatuses.putAll(statusMap); + if (!isOutsideMaxAge && value != null) { + rowValues.put(FeatureV2.getFeatureStringRef(feature.getFeatureReference()), value); + } else { + rowValues.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), + ValueProto.Value.newBuilder().build()); + } - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); + rowStatuses.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), + getMetadata(value, isOutsideMaxAge)); } else { - Map valueMap = - new HashMap<>() { - { - put( - FeatureV2.getFeatureStringRef(featureReference), - ValueProto.Value.newBuilder().build()); - } - }; - rowValues.putAll(valueMap); - - Map statusMap = - getMetadataMap(valueMap, true, false); - rowStatuses.putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); + rowValues.put( + FeatureV2.getFeatureStringRef(featureReference), + ValueProto.Value.newBuilder().build()); + + rowStatuses.put( + FeatureV2.getFeatureStringRef(featureReference), getMetadata(null, false)); } } + // Populate metrics/log request + populateCountMetrics(rowStatuses, projectName); } if (postProcessingSpan != null) { @@ -208,6 +239,71 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re populateHistogramMetrics(entityRows, featureReferences, projectName); populateFeatureCountMetrics(featureReferences, projectName); + // Handle ODFVs. For each ODFV reference, we send a TransformFeaturesRequest to the FTS. + // The request should contain the entity data, the retrieved features, and the request data. + if (!onDemandFeatureReferences.isEmpty()) { + // Augment values, which contains the entity data and retrieved features, with the request + // data. Also augment statuses. + for (int i = 0; i < values.size(); i++) { + Map rowValues = values.get(i); + Map rowStatuses = statuses.get(i); + + for (Map.Entry> entry : requestDataFeatures.entrySet()) { + String key = entry.getKey(); + List fieldValues = entry.getValue(); + rowValues.put(key, fieldValues.get(i)); + rowStatuses.put(key, GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + } + + // Serialize the augmented values. + ValueType transformationInput = + this.onlineTransformationService.serializeValuesIntoArrowIPC(values); + + // Send out requests to the FTS and process the responses. + Set onDemandFeatureStringReferences = + onDemandFeatureReferences.stream() + .map(r -> FeatureV2.getFeatureStringRef(r)) + .collect(Collectors.toSet()); + for (FeatureReferenceV2 featureReference : onDemandFeatureReferences) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + TransformFeaturesRequest transformFeaturesRequest = + TransformFeaturesRequest.newBuilder() + .setOnDemandFeatureViewName(onDemandFeatureViewName) + .setProject(projectName) + .setTransformationInput(transformationInput) + .build(); + + TransformFeaturesResponse transformFeaturesResponse = + this.onlineTransformationService.transformFeatures(transformFeaturesRequest); + + this.onlineTransformationService.processTransformFeaturesResponse( + transformFeaturesResponse, + onDemandFeatureViewName, + onDemandFeatureStringReferences, + values, + statuses); + } + + // Remove all features that were added as inputs for ODFVs. + Set addedFeatureStringReferences = + addedFeatureReferences.stream() + .map(r -> FeatureV2.getFeatureStringRef(r)) + .collect(Collectors.toSet()); + for (int i = 0; i < values.size(); i++) { + Map rowValues = values.get(i); + Map rowStatuses = statuses.get(i); + List keysToRemove = + rowValues.keySet().stream() + .filter(k -> addedFeatureStringReferences.contains(k)) + .collect(Collectors.toList()); + for (String key : keysToRemove) { + rowValues.remove(key); + rowStatuses.remove(key); + } + } + } + // Build response field values from entityValuesMap and entityStatusesMap // Response field values should be in the same order as the entityRows provided by the user. List fieldValuesList = @@ -219,20 +315,8 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re .putAllStatuses(statuses.get(entityRowIdx)) .build()) .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); - } - private boolean checkSameFeatureSpec( - ValueProto.ValueType.Enum valueTypeEnum, ValueProto.Value.ValCase valueCase) { - if (valueTypeEnum.equals(ValueProto.ValueType.Enum.INVALID)) { - return false; - } - - if (valueCase.equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return true; - } - - return TYPE_TO_VAL_CASE.get(valueTypeEnum).equals(valueCase); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } private static Map getFeatureRefFeatureMap(List features) { @@ -243,47 +327,23 @@ private static Map getFeatureRefFeatureMap(List getMetadataMap( - Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { - return valueMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - es -> { - ValueProto.Value fieldValue = es.getValue(); - if (isNotFound) { - return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND; - } else if (isOutsideMaxAge) { - return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE; - } else if (fieldValue.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE; - } - return GetOnlineFeaturesResponse.FieldStatus.PRESENT; - })); - } - - private static Map unpackValueMap( - Feature feature, boolean isOutsideMaxAge, boolean isMatchingFeatureSpec) { - Map valueMap = new HashMap<>(); - - if (!isOutsideMaxAge && isMatchingFeatureSpec) { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), feature.getFeatureValue()); - } else { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), - ValueProto.Value.newBuilder().build()); + private static GetOnlineFeaturesResponse.FieldStatus getMetadata( + ValueProto.Value value, boolean isOutsideMaxAge) { + + if (value == null) { + return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND; + } else if (isOutsideMaxAge) { + return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE; + } else if (value.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { + return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE; } - - return valueMap; + return GetOnlineFeaturesResponse.FieldStatus.PRESENT; } /** diff --git a/serving/src/main/java/feast/serving/service/OnlineTransformationService.java b/serving/src/main/java/feast/serving/service/OnlineTransformationService.java new file mode 100644 index 0000000..541fe46 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/OnlineTransformationService.java @@ -0,0 +1,412 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import com.google.protobuf.ByteString; +import feast.common.models.FeatureV2; +import feast.proto.core.DataSourceProto; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; +import feast.proto.serving.TransformationServiceGrpc; +import feast.proto.types.ValueProto; +import feast.serving.specs.FeatureSpecRetriever; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Status; +import java.io.IOException; +import java.nio.channels.Channels; +import java.util.*; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tomcat.util.http.fileupload.ByteArrayOutputStream; +import org.slf4j.Logger; + +public class OnlineTransformationService implements TransformationService { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(OnlineTransformationService.class); + private final TransformationServiceGrpc.TransformationServiceBlockingStub stub; + private final FeatureSpecRetriever featureSpecRetriever; + static final int INT64_BITWIDTH = 64; + static final int INT32_BITWIDTH = 32; + + public OnlineTransformationService( + String transformationServiceEndpoint, FeatureSpecRetriever featureSpecRetriever) { + if (transformationServiceEndpoint != null) { + final ManagedChannel channel = + ManagedChannelBuilder.forTarget(transformationServiceEndpoint).usePlaintext().build(); + this.stub = TransformationServiceGrpc.newBlockingStub(channel); + } else { + this.stub = null; + } + this.featureSpecRetriever = featureSpecRetriever; + } + + /** {@inheritDoc} */ + @Override + public TransformFeaturesResponse transformFeatures( + TransformFeaturesRequest transformFeaturesRequest) { + return this.stub.transformFeatures(transformFeaturesRequest); + } + + /** {@inheritDoc} */ + @Override + public Pair, List> + extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + List onDemandFeatureReferences, String projectName) { + Set requestDataFeatureNames = new HashSet(); + List onDemandFeatureInputs = + new ArrayList(); + for (ServingAPIProto.FeatureReferenceV2 featureReference : onDemandFeatureReferences) { + OnDemandFeatureViewProto.OnDemandFeatureViewSpec onDemandFeatureViewSpec = + this.featureSpecRetriever.getOnDemandFeatureViewSpec(projectName, featureReference); + Map inputs = + onDemandFeatureViewSpec.getInputsMap(); + + for (OnDemandFeatureViewProto.OnDemandInput input : inputs.values()) { + OnDemandFeatureViewProto.OnDemandInput.InputCase inputCase = input.getInputCase(); + switch (inputCase) { + case REQUEST_DATA_SOURCE: + DataSourceProto.DataSource requestDataSource = input.getRequestDataSource(); + DataSourceProto.DataSource.RequestDataOptions requestDataOptions = + requestDataSource.getRequestDataOptions(); + Set requestDataNames = requestDataOptions.getSchemaMap().keySet(); + requestDataFeatureNames.addAll(requestDataNames); + break; + case FEATURE_VIEW: + FeatureViewProto.FeatureView featureView = input.getFeatureView(); + FeatureViewProto.FeatureViewSpec featureViewSpec = featureView.getSpec(); + String featureViewName = featureViewSpec.getName(); + for (FeatureProto.FeatureSpecV2 featureSpec : featureViewSpec.getFeaturesList()) { + String featureName = featureSpec.getName(); + ServingAPIProto.FeatureReferenceV2 onDemandFeatureInput = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable(featureViewName) + .setName(featureName) + .build(); + onDemandFeatureInputs.add(onDemandFeatureInput); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "OnDemandInput proto input field has an unexpected type: " + inputCase) + .asRuntimeException(); + } + } + } + Pair, List> pair = + new ImmutablePair, List>( + requestDataFeatureNames, onDemandFeatureInputs); + return pair; + } + + /** {@inheritDoc} */ + public Pair< + List, + Map>> + separateEntityRows( + Set requestDataFeatureNames, ServingAPIProto.GetOnlineFeaturesRequestV2 request) { + // Separate entity rows into entity data and request feature data. + List entityRows = + new ArrayList(); + Map> requestDataFeatures = + new HashMap>(); + + for (ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow : + request.getEntityRowsList()) { + Map fieldsMap = new HashMap(); + + for (Map.Entry entry : entityRow.getFieldsMap().entrySet()) { + String key = entry.getKey(); + ValueProto.Value value = entry.getValue(); + + if (requestDataFeatureNames.contains(key)) { + if (!requestDataFeatures.containsKey(key)) { + requestDataFeatures.put(key, new ArrayList()); + } + requestDataFeatures.get(key).add(value); + } else { + fieldsMap.put(key, value); + } + } + + // Construct new entity row containing the extracted entity data, if necessary. + if (!fieldsMap.isEmpty()) { + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow newEntityRow = + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(entityRow.getTimestamp()) + .putAllFields(fieldsMap) + .build(); + entityRows.add(newEntityRow); + } + } + + Pair< + List, + Map>> + pair = + new ImmutablePair< + List, + Map>>(entityRows, requestDataFeatures); + return pair; + } + + /** {@inheritDoc} */ + public void processTransformFeaturesResponse( + feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse + transformFeaturesResponse, + String onDemandFeatureViewName, + Set onDemandFeatureStringReferences, + List> values, + List> statuses) { + try { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ArrowFileReader reader = + new ArrowFileReader( + new ByteArrayReadableSeekableByteChannel( + transformFeaturesResponse + .getTransformationOutput() + .getArrowValue() + .toByteArray()), + allocator); + reader.loadNextBatch(); + VectorSchemaRoot readBatch = reader.getVectorSchemaRoot(); + Schema responseSchema = readBatch.getSchema(); + List responseFields = responseSchema.getFields(); + + for (Field field : responseFields) { + String columnName = field.getName(); + String fullFeatureName = onDemandFeatureViewName + ":" + columnName; + ArrowType columnType = field.getType(); + + // The response will contain all features for the specified ODFV, so we + // skip the features that were not requested. + if (!onDemandFeatureStringReferences.contains(fullFeatureName)) { + continue; + } + + FieldVector fieldVector = readBatch.getVector(field); + int valueCount = fieldVector.getValueCount(); + + // TODO: support all Feast types + // TODO: clean up the switch statement + if (columnType instanceof ArrowType.Int) { + int bitWidth = ((ArrowType.Int) columnType).getBitWidth(); + switch (bitWidth) { + case INT64_BITWIDTH: + for (int i = 0; i < valueCount; i++) { + long int64Value = ((BigIntVector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setInt64Val(int64Value).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + case INT32_BITWIDTH: + for (int i = 0; i < valueCount; i++) { + int intValue = ((IntVector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setInt32Val(intValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " is of type ArrowType.Int but has bitWidth " + + bitWidth + + " which cannot be handled.") + .asRuntimeException(); + } + } else if (columnType instanceof ArrowType.FloatingPoint) { + FloatingPointPrecision precision = ((ArrowType.FloatingPoint) columnType).getPrecision(); + switch (precision) { + case DOUBLE: + for (int i = 0; i < valueCount; i++) { + double doubleValue = ((Float8Vector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setDoubleVal(doubleValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + case SINGLE: + for (int i = 0; i < valueCount; i++) { + float floatValue = ((Float4Vector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setFloatVal(floatValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " is of type ArrowType.FloatingPoint but has precision " + + precision + + " which cannot be handled.") + .asRuntimeException(); + } + } + } + } catch (IOException e) { + log.info(e.toString()); + throw Status.INTERNAL + .withDescription( + "Unable to correctly process transform features response: " + e.toString()) + .asRuntimeException(); + } + } + + /** {@inheritDoc} */ + public ValueType serializeValuesIntoArrowIPC(List> values) { + // In order to be serialized correctly, the data must be packaged in a VectorSchemaRoot. + // We first construct all the columns. + Map columnNameToColumn = new HashMap(); + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + Map firstAugmentedRowValues = values.get(0); + for (Map.Entry entry : firstAugmentedRowValues.entrySet()) { + // The Python FTS does not expect full feature names, so we extract the feature name. + String columnName = FeatureV2.getFeatureName(entry.getKey()); + ValueProto.Value.ValCase valCase = entry.getValue().getValCase(); + FieldVector column; + // TODO: support all Feast types + switch (valCase) { + case INT32_VAL: + column = new IntVector(columnName, allocator); + break; + case INT64_VAL: + column = new BigIntVector(columnName, allocator); + break; + case DOUBLE_VAL: + column = new Float8Vector(columnName, allocator); + break; + case FLOAT_VAL: + column = new Float4Vector(columnName, allocator); + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + columnName + " has a type that is currently not handled: " + valCase) + .asRuntimeException(); + } + column.allocateNew(); + columnNameToColumn.put(columnName, column); + } + + // Add the data, row by row. + for (int i = 0; i < values.size(); i++) { + Map augmentedRowValues = values.get(i); + + for (Map.Entry entry : augmentedRowValues.entrySet()) { + String columnName = FeatureV2.getFeatureName(entry.getKey()); + ValueProto.Value value = entry.getValue(); + ValueProto.Value.ValCase valCase = value.getValCase(); + FieldVector column = columnNameToColumn.get(columnName); + // TODO: support all Feast types + switch (valCase) { + case INT32_VAL: + ((IntVector) column).setSafe(i, value.getInt32Val()); + break; + case INT64_VAL: + ((BigIntVector) column).setSafe(i, value.getInt64Val()); + break; + case DOUBLE_VAL: + ((Float8Vector) column).setSafe(i, value.getDoubleVal()); + break; + case FLOAT_VAL: + ((Float4Vector) column).setSafe(i, value.getFloatVal()); + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " has a type that is currently not handled: " + + valCase) + .asRuntimeException(); + } + } + } + + // Construct the VectorSchemaRoot. + List columnFields = new ArrayList(); + List columns = new ArrayList(); + for (FieldVector column : columnNameToColumn.values()) { + column.setValueCount(values.size()); + columnFields.add(column.getField()); + columns.add(column); + } + VectorSchemaRoot schemaRoot = new VectorSchemaRoot(columnFields, columns); + + // Serialize the VectorSchemaRoot into Arrow IPC format. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowFileWriter writer = new ArrowFileWriter(schemaRoot, null, Channels.newChannel(out)); + try { + writer.start(); + writer.writeBatch(); + writer.end(); + } catch (IOException e) { + log.info(e.toString()); + throw Status.INTERNAL + .withDescription( + "ArrowFileWriter could not write properly; failed with error: " + e.toString()) + .asRuntimeException(); + } + byte[] byteData = out.toByteArray(); + ByteString inputData = ByteString.copyFrom(byteData); + ValueType transformationInput = ValueType.newBuilder().setArrowValue(inputData).build(); + return transformationInput; + } +} diff --git a/serving/src/main/java/feast/serving/service/TransformationService.java b/serving/src/main/java/feast/serving/service/TransformationService.java new file mode 100644 index 0000000..caa5279 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/TransformationService.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; +import feast.proto.types.ValueProto; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; + +public interface TransformationService { + /** + * Apply on demand transformations for the specified ODFVs. + * + * @param transformFeaturesRequest proto containing the ODFV references and necessary data + * @return a proto object containing the response + */ + TransformFeaturesResponse transformFeatures(TransformFeaturesRequest transformFeaturesRequest); + + /** + * Extract the set of request data feature names and the list of on demand feature inputs from a + * list of ODFV references. + * + * @param onDemandFeatureReferences list of ODFV references to be parsed + * @param projectName project name + * @return a pair containing the set of request data feature names and list of on demand feature + * inputs + */ + Pair, List> + extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + List onDemandFeatureReferences, String projectName); + + /** + * Separate the entity rows of a request into entity data and request feature data. + * + * @param requestDataFeatureNames set of feature names for the request data + * @param request the GetOnlineFeaturesRequestV2 containing the entity rows + * @return a pair containing the set of request data feature names and list of on demand feature + * inputs + */ + Pair, Map>> + separateEntityRows(Set requestDataFeatureNames, GetOnlineFeaturesRequestV2 request); + + /** + * Process a response from the feature transformation server by augmenting the given lists of + * field maps and status maps with the correct fields from the response. + * + * @param transformFeaturesResponse response to be processed + * @param onDemandFeatureViewName name of ODFV to which the response corresponds + * @param onDemandFeatureStringReferences set of all ODFV references that should be kept + * @param values list of field maps to be augmented with additional fields from the response + * @param statuses list of status maps to be augmented + */ + void processTransformFeaturesResponse( + TransformFeaturesResponse transformFeaturesResponse, + String onDemandFeatureViewName, + Set onDemandFeatureStringReferences, + List> values, + List> statuses); + + /** + * Serialize data into Arrow IPC format, to be sent to the Python feature transformation server. + * + * @param values list of field maps to be serialized + * @return the data packaged into a ValueType proto object + */ + ValueType serializeValuesIntoArrowIPC(List> values); +} diff --git a/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java new file mode 100644 index 0000000..2a88659 --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public class CoreFeatureSpecRetriever implements FeatureSpecRetriever { + private final CachedSpecService specService; + + public CoreFeatureSpecRetriever(CachedSpecService specService) { + this.specService = specService; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getMaxAge(); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getEntitiesList(); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureSpec(projectName, featureReference); + } + + @Override + public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + throw new UnsupportedOperationException( + String.format("Feast Core does not support getting feature view specs.")); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + throw new UnsupportedOperationException( + String.format("Feast Core does not support on demand feature views.")); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + return false; + } +} diff --git a/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java new file mode 100644 index 0000000..57e931c --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public interface FeatureSpecRetriever { + + Duration getMaxAge(String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java new file mode 100644 index 0000000..0cd851e --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import feast.serving.registry.RegistryRepository; +import java.util.List; + +public class RegistryFeatureSpecRetriever implements FeatureSpecRetriever { + private final RegistryRepository registryRepository; + + public RegistryFeatureSpecRetriever(RegistryRepository registryRepository) { + this.registryRepository = registryRepository; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getTtl(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getEntitiesList(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getFeatureSpec(projectName, featureReference); + } + + @Override + public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getFeatureViewSpec(projectName, featureReference); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getOnDemandFeatureViewSpec(projectName, featureReference); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.isOnDemandFeatureReference(featureReference); + } +} diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 90b9493..dca2b5e 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -26,7 +26,7 @@ public class Metrics { .name("request_latency_seconds") .subsystem("feast_serving") .help("Request latency in seconds") - .labelNames("method") + .labelNames("method", "project") .register(); public static final Histogram requestEntityCountDistribution = diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index b20fd8e..3e4e07b 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -3,7 +3,7 @@ feast: # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: ${FEAST_CORE_HOST:localhost} core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - + core-authentication: enabled: false # should be set to true if authentication is enabled on core. provider: google # can be set to `oauth` or `google` @@ -54,6 +54,18 @@ feast: read_from: MASTER # Redis operation timeout in ISO-8601 format timeout: PT0.5S + - name: bigtable + type: BIGTABLE + config: + project_id: + instance_id: + app_profile_id: + - name: cassandra + type: CASSANDRA + config: + connection_string: localhost:9042 + data_center: datacenter1 + keyspace: feast tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls # which can be useful to debug performance issues and perform benchmarking diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 79d4773..d49ac41 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -51,6 +51,16 @@ public class BaseAuthIT { static final String REDIS = "redis_1"; static final int REDIS_PORT = 6379; + static final String BIGTABLE = "bigtable_1"; + static final int BIGTABLE_PORT = 8086; + + static final String CASSANDRA = "cassandra_1"; + static final int CASSANDRA_PORT = 9042; + static final String CASSANDRA_DATACENTER = "datacenter1"; + static final String CASSANDRA_KEYSPACE = "feast"; + static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference"; + static final String CASSANDRA_ENTITY_KEY = "key"; + static final int FEAST_CORE_PORT = 6565; @DynamicPropertySource @@ -69,8 +79,39 @@ static void properties(DynamicPropertyRegistry registry) { } }); registry.add("feast.stores[0].config.port", () -> REDIS_PORT); - registry.add("feast.stores[0].subscriptions[0].name", () -> "*"); - registry.add("feast.stores[0].subscriptions[0].project", () -> "*"); + + registry.add("feast.stores[1].name", () -> "bigtable"); + registry.add("feast.stores[1].type", () -> "BIGTABLE"); + registry.add("feast.stores[1].config.project_id", () -> "test-project"); + registry.add("feast.stores[1].config.instance_id", () -> "test-instance"); + + registry.add("feast.stores[2].name", () -> "cassandra"); + registry.add("feast.stores[2].type", () -> "CASSANDRA"); + registry.add( + "feast.stores[2].config.host", + () -> { + try { + return InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + return ""; + } + }); + + registry.add( + "feast.stores[2].config.connection_string", + () -> { + String hostAddress = ""; + try { + hostAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + return String.format("%s:%s", hostAddress, CASSANDRA_PORT); + }); + registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); + registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE); registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java b/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java new file mode 100644 index 0000000..21bddb5 --- /dev/null +++ b/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java @@ -0,0 +1,873 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.stub.BigtableTableAdminStubSettings; +import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.protobuf.ByteString; +import feast.common.it.DataGenerator; +import feast.common.models.FeatureV2; +import feast.proto.core.EntityProto; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingServiceGrpc; +import feast.proto.types.ValueProto; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@ActiveProfiles("it") +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "feast.core-cache-refresh-interval=1", + "feast.active_store=bigtable", + "spring.main.allow-bean-definition-overriding=true" + }) +@Testcontainers +public class ServingServiceBigTableIT extends BaseAuthIT { + + static final Map options = new HashMap<>(); + static CoreSimpleAPIClient coreClient; + static ServingServiceGrpc.ServingServiceBlockingStub servingStub; + + static BigtableDataClient client; + static final int FEAST_SERVING_PORT = 6569; + + static final String PROJECT_ID = "test-project"; + static final String INSTANCE_ID = "test-instance"; + static final String APP_PROFILE_ID = "default"; + static ManagedChannel channel; + + static final FeatureReferenceV2 feature1Reference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + static final FeatureReferenceV2 feature2Reference = + DataGenerator.createFeatureReference("rides", "trip_distance"); + static final FeatureReferenceV2 feature3Reference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + static final FeatureReferenceV2 feature4Reference = + DataGenerator.createFeatureReference("rides", "trip_wrong_type"); + + @ClassRule @Container + public static DockerComposeContainer environment = + new DockerComposeContainer( + new File("src/test/resources/docker-compose/docker-compose-bigtable-it.yml")) + .withExposedService( + CORE, + FEAST_CORE_PORT, + Wait.forLogMessage(".*gRPC Server started.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) + .withExposedService(BIGTABLE, BIGTABLE_PORT); + + @DynamicPropertySource + static void initialize(DynamicPropertyRegistry registry) { + registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + } + + @BeforeAll + static void globalSetup() throws IOException { + coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); + servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); + + // Initialize BigTable Client + client = + BigtableDataClient.create( + BigtableDataSettings.newBuilderForEmulator( + environment.getServiceHost("bigtable_1", BIGTABLE_PORT), + environment.getServicePort("bigtable_1", BIGTABLE_PORT)) + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .build()); + + String endpoint = + environment.getServiceHost("bigtable_1", BIGTABLE_PORT) + + ":" + + environment.getServicePort("bigtable_1", BIGTABLE_PORT); + channel = ManagedChannelBuilder.forTarget(endpoint).usePlaintext().build(); + + /** Feast resource creation Workflow */ + String projectName = "default"; + // Apply Entity (driver_id) + String driverEntityName = "driver_id"; + String driverEntityDescription = "My driver id"; + ValueProto.ValueType.Enum driverEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 driverEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(driverEntityName) + .setDescription(driverEntityDescription) + .setValueType(driverEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, driverEntitySpec); + + // Apply Entity (this_is_a_long_long_long_long_long_long_entity_id) + String superLongEntityName = "this_is_a_long_long_long_long_long_long_entity_id"; + String superLongEntityDescription = "My super long entity id"; + ValueProto.ValueType.Enum superLongEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 superLongEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(superLongEntityName) + .setDescription(superLongEntityDescription) + .setValueType(superLongEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, superLongEntitySpec); + + // Apply Entity (merchant_id) + String merchantEntityName = "merchant_id"; + String merchantEntityDescription = "My merchant id"; + ValueProto.ValueType.Enum merchantEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 merchantEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(merchantEntityName) + .setDescription(merchantEntityDescription) + .setValueType(merchantEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, merchantEntitySpec); + + // Apply FeatureTable (rides) + String ridesFeatureTableName = "rides"; + ImmutableList ridesEntities = ImmutableList.of(driverEntityName); + ImmutableMap ridesFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT64, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE, + "trip_empty", + ValueProto.ValueType.Enum.DOUBLE, + "trip_wrong_type", + ValueProto.ValueType.Enum.STRING); + TestUtils.applyFeatureTable( + coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); + + // Apply FeatureTable (superLong) + String superLongFeatureTableName = "superlong"; + ImmutableList superLongEntities = ImmutableList.of(superLongEntityName); + ImmutableMap superLongFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT64, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE, + "trip_empty", + ValueProto.ValueType.Enum.DOUBLE, + "trip_wrong_type", + ValueProto.ValueType.Enum.STRING); + TestUtils.applyFeatureTable( + coreClient, + projectName, + superLongFeatureTableName, + superLongEntities, + superLongFeatures, + 7200); + + // Apply FeatureTable (rides_merchant) + String rideMerchantFeatureTableName = "rides_merchant"; + ImmutableList ridesMerchantEntities = + ImmutableList.of(driverEntityName, merchantEntityName); + TestUtils.applyFeatureTable( + coreClient, + projectName, + rideMerchantFeatureTableName, + ridesMerchantEntities, + ridesFeatures, + 7200); + + // BigTable Table names + String superLongBtTableName = String.format("%s__%s", projectName, superLongEntityName); + String hashSuffix = + Hashing.murmur3_32().hashBytes(superLongBtTableName.substring(42).getBytes()).toString(); + superLongBtTableName = + superLongBtTableName + .substring(0, Math.min(superLongBtTableName.length(), 42)) + .concat(hashSuffix); + String btTableName = String.format("%s__%s", projectName, driverEntityName); + String compoundBtTableName = + String.format( + "%s__%s", + projectName, ridesMerchantEntities.stream().collect(Collectors.joining("__"))); + String featureTableName = "rides"; + String metadataColumnFamily = "metadata"; + ImmutableList columnFamilies = ImmutableList.of(featureTableName, metadataColumnFamily); + ImmutableList compoundColumnFamilies = + ImmutableList.of(rideMerchantFeatureTableName, metadataColumnFamily); + + /** Single Entity Ingestion Workflow */ + Schema ftSchema = + SchemaBuilder.record("DriverData") + .namespace(featureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] schemaReference = + Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); + + GenericRecord record = + new GenericRecordBuilder(ftSchema) + .set("trip_cost", 5L) + .set("trip_distance", 3.5) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] entityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] entityFeatureValue = createEntityValue(ftSchema, schemaReference, record); + byte[] schemaKey = createSchemaKey(schemaReference); + ingestData( + featureTableName, btTableName, entityFeatureKey, entityFeatureValue, schemaKey, ftSchema); + + /** SuperLong Entity Ingestion Workflow */ + Schema superLongFtSchema = + SchemaBuilder.record("SuperLongData") + .namespace(superLongFeatureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] superLongSchemaReference = + Hashing.murmur3_32().hashBytes(superLongFtSchema.toString().getBytes()).asBytes(); + + GenericRecord superLongRecord = + new GenericRecordBuilder(superLongFtSchema) + .set("trip_cost", 5L) + .set("trip_distance", 3.5) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] superLongEntityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] superLongEntityFeatureValue = + createEntityValue(superLongFtSchema, superLongSchemaReference, superLongRecord); + byte[] superLongSchemaKey = createSchemaKey(superLongSchemaReference); + ingestData( + superLongFeatureTableName, + superLongBtTableName, + superLongEntityFeatureKey, + superLongEntityFeatureValue, + superLongSchemaKey, + superLongFtSchema); + + /** Compound Entity Ingestion Workflow */ + Schema compoundFtSchema = + SchemaBuilder.record("DriverMerchantData") + .namespace(rideMerchantFeatureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] compoundSchemaReference = + Hashing.murmur3_32().hashBytes(compoundFtSchema.toString().getBytes()).asBytes(); + + // Entity-Feature Row + GenericRecord compoundEntityRecord = + new GenericRecordBuilder(compoundFtSchema) + .set("trip_cost", 10L) + .set("trip_distance", 5.5) + .set("trip_empty", null) + .set("trip_wrong_type", "wrong_type") + .build(); + + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + byte[] compoundEntityFeatureKey = + ridesMerchantEntities.stream() + .map(entity -> DataGenerator.valueToString(entityRow.getFieldsMap().get(entity))) + .collect(Collectors.joining("#")) + .getBytes(); + byte[] compoundEntityFeatureValue = + createEntityValue(compoundFtSchema, compoundSchemaReference, compoundEntityRecord); + byte[] compoundSchemaKey = createSchemaKey(compoundSchemaReference); + ingestData( + rideMerchantFeatureTableName, + compoundBtTableName, + compoundEntityFeatureKey, + compoundEntityFeatureValue, + compoundSchemaKey, + compoundFtSchema); + + // set up options for call credentials + options.put("oauth_url", TOKEN_URL); + options.put(CLIENT_ID, CLIENT_ID); + options.put(CLIENT_SECRET, CLIENT_SECRET); + options.put("jwkEndpointURI", JWK_URI); + options.put("audience", AUDIENCE); + options.put("grant_type", GRANT_TYPE); + } + + @AfterAll + static void tearDown() { + ((ManagedChannel) servingStub.getChannel()).shutdown(); + channel.shutdown(); + } + + private static void createTable( + TransportChannelProvider channelProvider, + CredentialsProvider credentialsProvider, + String tableName, + List columnFamilies) + throws IOException { + EnhancedBigtableTableAdminStub stub = + EnhancedBigtableTableAdminStub.createEnhanced( + BigtableTableAdminStubSettings.newBuilder() + .setTransportChannelProvider(channelProvider) + .setCredentialsProvider(credentialsProvider) + .build()); + + try (BigtableTableAdminClient client = + BigtableTableAdminClient.create(PROJECT_ID, INSTANCE_ID, stub)) { + CreateTableRequest createTableRequest = CreateTableRequest.of(tableName); + for (String columnFamily : columnFamilies) { + createTableRequest.addFamily(columnFamily); + } + if (!client.exists(tableName)) { + client.createTable(createTableRequest); + } + } + } + + private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { + String schemaKeyPrefix = "schema#"; + + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaKeyPrefix.getBytes()); + concatOutputStream.write(schemaReference); + byte[] schemaKey = concatOutputStream.toByteArray(); + + return schemaKey; + } + + private static byte[] createEntityValue( + Schema schema, byte[] schemaReference, GenericRecord record) throws IOException { + // Entity-Feature Row + byte[] avroSerializedFeatures = recordToAvro(record, schema); + + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaReference); + concatOutputStream.write("".getBytes()); + concatOutputStream.write(avroSerializedFeatures); + byte[] entityFeatureValue = concatOutputStream.toByteArray(); + + return entityFeatureValue; + } + + private static byte[] schemaReference(Schema schema) { + return Hashing.murmur3_32().hashBytes(schema.toString().getBytes()).asBytes(); + } + + private static void ingestData( + String featureTableName, + String btTableName, + byte[] btEntityFeatureKey, + byte[] btEntityFeatureValue, + byte[] btSchemaKey, + Schema btSchema) + throws IOException { + String emptyQualifier = ""; + String avroQualifier = "avro"; + String metadataColumnFamily = "metadata"; + + TransportChannelProvider channelProvider = + FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); + NoCredentialsProvider credentialsProvider = NoCredentialsProvider.create(); + createTable( + channelProvider, + credentialsProvider, + btTableName, + ImmutableList.of(featureTableName, metadataColumnFamily)); + + // Update Compound Entity-Feature Row + client.mutateRow( + RowMutation.create(btTableName, ByteString.copyFrom(btEntityFeatureKey)) + .setCell( + featureTableName, + ByteString.copyFrom(emptyQualifier.getBytes()), + ByteString.copyFrom(btEntityFeatureValue))); + + // Update Schema Row + client.mutateRow( + RowMutation.create(btTableName, ByteString.copyFrom(btSchemaKey)) + .setCell( + metadataColumnFamily, + ByteString.copyFrom(avroQualifier.getBytes()), + ByteString.copyFrom(btSchema.toString().getBytes()))); + } + + private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); + writer.write(datum, encoder); + encoder.flush(); + + return output.toByteArray(); + } + + @Test + public void shouldRegisterSingleEntityAndGetOnlineFeatures() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); + ImmutableList entityRows = ImmutableList.of(entityRow1); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { + String projectName = "default"; + String driverEntityName = "driver_id"; + String merchantEntityName = "merchant_id"; + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + ImmutableList entityRows = ImmutableList.of(entityRow); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + driverEntityName, + driverEntityValue, + merchantEntityName, + merchantEntityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + driverEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + merchantEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldReturnCorrectRowCount() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, entityValue1, 100); + GetOnlineFeaturesRequestV2.EntityRow entityRow2 = + DataGenerator.createEntityRow(entityName, entityValue2, 100); + ImmutableList entityRows = + ImmutableList.of(entityRow1, entityRow2); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + FeatureReferenceV2 emptyFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue1, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + + ImmutableMap expectedValueMap2 = + ImmutableMap.of( + entityName, + entityValue2, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap2 = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap2) + .putAllStatuses(expectedStatusMap2) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues, expectedFieldValues2); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldSupportAllFeastTypes() throws IOException { + EntityProto.EntitySpecV2 entitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName("entity") + .setDescription("") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build(); + TestUtils.applyEntity(coreClient, "default", entitySpec); + + ImmutableMap allTypesFeatures = + new ImmutableMap.Builder() + .put("f_int64", ValueProto.ValueType.Enum.INT64) + .put("f_int32", ValueProto.ValueType.Enum.INT32) + .put("f_float", ValueProto.ValueType.Enum.FLOAT) + .put("f_double", ValueProto.ValueType.Enum.DOUBLE) + .put("f_string", ValueProto.ValueType.Enum.STRING) + .put("f_bytes", ValueProto.ValueType.Enum.BYTES) + .put("f_bool", ValueProto.ValueType.Enum.BOOL) + .put("f_int64_list", ValueProto.ValueType.Enum.INT64_LIST) + .put("f_int32_list", ValueProto.ValueType.Enum.INT32_LIST) + .put("f_float_list", ValueProto.ValueType.Enum.FLOAT_LIST) + .put("f_double_list", ValueProto.ValueType.Enum.DOUBLE_LIST) + .put("f_string_list", ValueProto.ValueType.Enum.STRING_LIST) + .put("f_bytes_list", ValueProto.ValueType.Enum.BYTES_LIST) + .put("f_bool_list", ValueProto.ValueType.Enum.BOOL_LIST) + .build(); + + TestUtils.applyFeatureTable( + coreClient, "default", "all_types", ImmutableList.of("entity"), allTypesFeatures, 7200); + + Schema schema = + SchemaBuilder.record("AllTypesRecord") + .namespace("") + .fields() + .requiredLong("f_int64") + .requiredInt("f_int32") + .requiredFloat("f_float") + .requiredDouble("f_double") + .requiredString("f_string") + .requiredBytes("f_bytes") + .requiredBoolean("f_bool") + .name("f_int64_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().longType())) + .noDefault() + .name("f_int32_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().intType())) + .noDefault() + .name("f_float_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().floatType())) + .noDefault() + .name("f_double_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().doubleType())) + .noDefault() + .name("f_string_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())) + .noDefault() + .name("f_bytes_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().bytesType())) + .noDefault() + .name("f_bool_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().booleanType())) + .noDefault() + .endRecord(); + + GenericData.Record record = + new GenericRecordBuilder(schema) + .set("f_int64", 10L) + .set("f_int32", 10) + .set("f_float", 10.0) + .set("f_double", 10.0D) + .set("f_string", "test") + .set("f_bytes", ByteBuffer.wrap("test".getBytes())) + .set("f_bool", true) + .set("f_int64_list", ImmutableList.of(10L)) + .set("f_int32_list", ImmutableList.of(10)) + .set("f_float_list", ImmutableList.of(10.0)) + .set("f_double_list", ImmutableList.of(10.0D)) + .set("f_string_list", ImmutableList.of("test")) + .set("f_bytes_list", ImmutableList.of(ByteBuffer.wrap("test".getBytes()))) + .set("f_bool_list", ImmutableList.of(true)) + .build(); + + ValueProto.Value entity = DataGenerator.createStrValue("key"); + + ingestData( + "all_types", + "default__entity", + entity.getStringVal().getBytes(), + createEntityValue(schema, schemaReference(schema), record), + createSchemaKey(schemaReference(schema)), + schema); + + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest( + "default", + allTypesFeatures.keySet().stream() + .map( + f -> + FeatureReferenceV2.newBuilder() + .setFeatureTable("all_types") + .setName(f) + .build()) + .collect(Collectors.toList()), + ImmutableList.of(DataGenerator.createEntityRow("entity", entity, 100))); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + assert featureResponse.getFieldValues(0).getStatusesMap().values().stream() + .allMatch(status -> status.equals(GetOnlineFeaturesResponse.FieldStatus.PRESENT)); + } + + @Test + public void shouldRegisterSuperLongEntityAndGetOnlineFeatures() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "this_is_a_long_long_long_long_long_long_entity_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); + ImmutableList entityRows = ImmutableList.of(entityRow1); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("superlong", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("superlong", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @TestConfiguration + public static class TestConfig { + @Bean + public BigtableDataClient bigtableClient() throws IOException { + return BigtableDataClient.create( + BigtableDataSettings.newBuilderForEmulator( + environment.getServiceHost("bigtable_1", BIGTABLE_PORT), + environment.getServicePort("bigtable_1", BIGTABLE_PORT)) + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setAppProfileId(APP_PROFILE_ID) + .build()); + } + } +} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceIT.java b/serving/src/test/java/feast/serving/it/ServingServiceIT.java deleted file mode 100644 index f08ff88..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceIT.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.*; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.storage.RedisProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.*; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - }) -@Testcontainers -public class ServingServiceIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static final String timestampPrefix = "_ts"; - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - static RedisCommands syncCommands; - - static final int FEAST_SERVING_PORT = 6568; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(REDIS, REDIS_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - RedisClient redisClient = - RedisClient.create( - new RedisURI( - environment.getServiceHost("redis_1", REDIS_PORT), - environment.getServicePort("redis_1", REDIS_PORT), - java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - syncCommands = connection.sync(); - - String projectName = "default"; - // Apply Entity - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - String description = "My driver id"; - ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(entityName) - .setDescription(description) - .setValueType(entityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, entitySpec); - - // Apply FeatureTable - String featureTableName = "rides"; - ImmutableList entities = ImmutableList.of(entityName); - - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - ServingAPIProto.FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - // Event Timestamp - String eventTimestampKey = timestampPrefix + ":" + featureTableName; - Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); - - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Serialize Redis Key with Entity i.e - RedisProto.RedisKeyV2 redisKey = - RedisProto.RedisKeyV2.newBuilder() - .setProject(projectName) - .addEntityNames(entityName) - .addEntityValues(entityValue) - .build(); - - ImmutableMap featureReferenceValueMap = - ImmutableMap.of( - feature1Reference, - DataGenerator.createInt64Value(42), - feature2Reference, - DataGenerator.createDoubleValue(42.2), - feature3Reference, - DataGenerator.createEmptyValue(), - feature4Reference, - DataGenerator.createDoubleValue(42.2)); - - // Insert timestamp into Redis and isTimestampMap only once - syncCommands.hset( - redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); - featureReferenceValueMap.forEach( - (featureReference, featureValue) -> { - // Murmur hash Redis Feature Field i.e murmur() - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - byte[] featureReferenceBytes = - Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - // Insert features into Redis - syncCommands.hset( - redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); - }); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldRegisterAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ImmutableList featureReferences = - ImmutableList.of(feature1Reference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(feature1Reference), - DataGenerator.createInt64Value(42)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(feature1Reference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldRegisterAndGetOnlineFeaturesWithNotFound() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - ServingAPIProto.FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(42), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldGetOnlineFeaturesOutsideMaxAge() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 7400); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForDiffType() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForUpdatedType() { - String projectName = "default"; - String entityName = "driver_id"; - String featureTableName = "rides"; - - ImmutableList entities = ImmutableList.of(entityName); - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.STRING, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Sleep is necessary to ensure caching (every 1s) of updated FeatureTable is done - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - } - - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java deleted file mode 100644 index 8f2440d..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import com.google.common.collect.ImmutableMap; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false" - }) -@Testcontainers -public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { - - CoreSimpleAPIClient coreClient; - FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; - static final Map options = new HashMap<>(); - - static final int FEAST_SERVING_PORT = 6566; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @BeforeEach - public void initState() { - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - EntityProto.EntitySpecV2 entitySpec = - DataGenerator.createEntitySpecV2( - ENTITY_ID, - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); - - expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - Arrays.asList(ENTITY_ID), - new HashMap<>() { - { - put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldAllowUnauthenticatedGetOnlineFeatures() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); - - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAuthenticated() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java deleted file mode 100644 index 64fe44b..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import feast.common.it.DataGenerator; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import sh.ory.keto.ApiException; - -@ActiveProfiles("it") -@SpringBootTest( - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true" - }) -@Testcontainers -public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { - - static final Map adminCredentials = new HashMap<>(); - static final Map memberCredentials = new HashMap<>(); - static final String PROJECT_MEMBER_CLIENT_ID = "client_id_1"; - static final String NOT_PROJECT_MEMBER_CLIENT_ID = "client_id_2"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - static String subjectClaim = "sub"; - static CoreSimpleAPIClient coreClient; - static final int FEAST_SERVING_PORT = 6766; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml"), - new File("src/test/resources/docker-compose/docker-compose-it-keto.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - AuthTestUtils.seedKeto(ketoExternalUrl, PROJECT_NAME, PROJECT_MEMBER_CLIENT_ID, CLIENT_ID); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, NOT_PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - // set up options for call credentials - adminCredentials.put("oauth_url", TOKEN_URL); - adminCredentials.put(CLIENT_ID, CLIENT_ID); - adminCredentials.put(CLIENT_SECRET, CLIENT_SECRET); - adminCredentials.put("jwkEndpointURI", JWK_URI); - adminCredentials.put("audience", AUDIENCE); - adminCredentials.put("grant_type", GRANT_TYPE); - - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); - coreClient.simpleApplyEntity( - PROJECT_NAME, - DataGenerator.createEntitySpecV2( - ENTITY_ID, "", ValueProto.ValueType.Enum.STRING, Collections.emptyMap())); - coreClient.simpleApplyFeatureTable( - PROJECT_NAME, - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - ImmutableList.of(ENTITY_ID), - ImmutableMap.of(FEATURE_NAME, ValueProto.ValueType.Enum.STRING), - 0, - Collections.emptyMap())); - } - - @Test - public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAdmin() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfProjectMember() { - Map memberCredsOptions = new HashMap<>(); - memberCredsOptions.putAll(adminCredentials); - memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void cantGetOnlineFeaturesIfNotProjectMember() { - Map notMemberCredsOptions = new HashMap<>(); - notMemberCredsOptions.putAll(adminCredentials); - notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", - PROJECT_NAME, NOT_PROJECT_MEMBER_CLIENT_ID); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 539ed39..d3e62b4 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -34,7 +34,9 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.ProtoFeature; import feast.storage.connectors.redis.retriever.OnlineRetriever; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; @@ -51,6 +53,7 @@ public class OnlineServingServiceTest { @Mock CachedSpecService specService; @Mock Tracer tracer; @Mock OnlineRetriever retrieverV2; + private String transformationServiceEndpoint; private OnlineServingServiceV2 onlineServingServiceV2; @@ -60,69 +63,62 @@ public class OnlineServingServiceTest { @Before public void setUp() { initMocks(this); - onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); + CoreFeatureSpecRetriever coreFeatureSpecRetriever = new CoreFeatureSpecRetriever(specService); + OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, coreFeatureSpecRetriever); + onlineServingServiceV2 = + new OnlineServingServiceV2( + retrieverV2, tracer, coreFeatureSpecRetriever, onlineTransformationService); mockedFeatureRows = new ArrayList<>(); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("1")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("1"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("2")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("2"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("3")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("3"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("4")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("4"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_3") - .build()) - .setFeatureValue(createStrValue("5")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_3") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("5"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("6")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(50).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(50).build(), + createStrValue("6"))); featureSpecs = new ArrayList<>(); featureSpecs.add( @@ -163,7 +159,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); @@ -230,7 +226,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); @@ -294,7 +290,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())) .thenReturn( FeatureTableSpec.newBuilder() diff --git a/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml b/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml new file mode 100644 index 0000000..28985ef --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml @@ -0,0 +1,38 @@ +version: '3' + +services: + core: + image: gcr.io/kf-feast/feast-core:develop + volumes: + - ./core/application-it.yml:/etc/feast/application.yml + environment: + DB_HOST: db + restart: on-failure + depends_on: + - db + ports: + - 6565:6565 + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + + bigtable: + image: google/cloud-sdk:latest + environment: + GOOGLE_APPLICATION_CREDENTIALS: /Users/user/.config/gcloud/application_default_credentials.json + command: + - gcloud + - beta + - emulators + - bigtable + - start + - --host-port=0.0.0.0:8086 \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml new file mode 100644 index 0000000..15afad0 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml @@ -0,0 +1,31 @@ +version: '3' + +services: + core: + image: gcr.io/kf-feast/feast-core:develop + volumes: + - ./core/application-it.yml:/etc/feast/application.yml + environment: + DB_HOST: db + restart: on-failure + depends_on: + - db + ports: + - 6565:6565 + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + + cassandra: + image: datastax/cassandra:4.0 + ports: + - "9042:9042" \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml new file mode 100644 index 0000000..33d65f4 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + redis: + image: redis:5-alpine + ports: + - "6379:6379" + materialize: + build: feast10 + links: + - redis + diff --git a/serving/src/test/resources/docker-compose/feast10/Dockerfile b/serving/src/test/resources/docker-compose/feast10/Dockerfile new file mode 100644 index 0000000..bde9f11 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7 + +WORKDIR /usr/src/ + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "./materialize.py" ] diff --git a/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet b/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet new file mode 100644 index 0000000..df8cbba Binary files /dev/null and b/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet differ diff --git a/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml new file mode 100644 index 0000000..87e3310 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml @@ -0,0 +1,9 @@ +project: feast_project +provider: local +online_store: + type: redis + connection_string: "redis:6379" +offline_store: {} +flags: + alpha_features: true + on_demand_transforms: true diff --git a/serving/src/test/resources/docker-compose/feast10/materialize.py b/serving/src/test/resources/docker-compose/feast10/materialize.py new file mode 100644 index 0000000..6338c16 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/materialize.py @@ -0,0 +1,45 @@ +# This is an example feature definition file + +from google.protobuf.duration_pb2 import Duration + +from datetime import datetime +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, FeatureStore + +print("Running materialize.py") + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +file_path = "driver_stats.parquet" +driver_hourly_stats = FileSource( + path=file_path, + event_timestamp_column="event_timestamp", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=Duration(seconds=86400 * 365), + features=[ + Feature(name="conv_rate", dtype=ValueType.DOUBLE), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), + ], + online=True, + batch_source=driver_hourly_stats, + tags={}, +) + +fs = FeatureStore(".") +fs.apply([driver_hourly_stats_view, driver]) + +now = datetime.now() +fs.materialize_incremental(now) diff --git a/serving/src/test/resources/docker-compose/feast10/registry.db b/serving/src/test/resources/docker-compose/feast10/registry.db new file mode 100644 index 0000000..774b493 Binary files /dev/null and b/serving/src/test/resources/docker-compose/feast10/registry.db differ diff --git a/serving/src/test/resources/docker-compose/feast10/requirements.txt b/serving/src/test/resources/docker-compose/feast10/requirements.txt new file mode 100644 index 0000000..cb579a2 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/requirements.txt @@ -0,0 +1 @@ +feast[redis]>=0.13,<1 diff --git a/storage/api/pom.xml b/storage/api/pom.xml index cc2f84e..4e7ad39 100644 --- a/storage/api/pom.xml +++ b/storage/api/pom.xml @@ -61,6 +61,12 @@ 3.9 + + org.apache.avro + avro + 1.10.2 + + junit junit diff --git a/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java b/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java new file mode 100644 index 0000000..96f19cc --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.retriever; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; +import java.nio.ByteBuffer; +import java.util.stream.Collectors; +import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; + +public class AvroFeature implements Feature { + private final ServingAPIProto.FeatureReferenceV2 featureReference; + + private final Timestamp eventTimestamp; + + private final Object featureValue; + + public AvroFeature( + ServingAPIProto.FeatureReferenceV2 featureReference, + Timestamp eventTimestamp, + Object featureValue) { + this.featureReference = featureReference; + this.eventTimestamp = eventTimestamp; + this.featureValue = featureValue; + } + + /** + * Casts feature value of Object type based on Feast valueType. Empty object i.e new Object() is + * interpreted as VAL_NOT_SET Feast valueType. + * + * @param valueType Feast valueType of feature as specified in FeatureSpec + * @return ValueProto.Value representation of feature + */ + @Override + public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) { + ValueProto.Value finalValue; + + try { + switch (valueType) { + case STRING: + finalValue = + ValueProto.Value.newBuilder().setStringVal(((Utf8) featureValue).toString()).build(); + break; + case INT32: + finalValue = ValueProto.Value.newBuilder().setInt32Val((Integer) featureValue).build(); + break; + case INT64: + finalValue = ValueProto.Value.newBuilder().setInt64Val((Long) featureValue).build(); + break; + case DOUBLE: + finalValue = ValueProto.Value.newBuilder().setDoubleVal((Double) featureValue).build(); + break; + case FLOAT: + finalValue = ValueProto.Value.newBuilder().setFloatVal((Float) featureValue).build(); + break; + case BYTES: + finalValue = + ValueProto.Value.newBuilder() + .setBytesVal(ByteString.copyFrom(((ByteBuffer) featureValue).array())) + .build(); + break; + case BOOL: + finalValue = ValueProto.Value.newBuilder().setBoolVal((Boolean) featureValue).build(); + break; + case STRING_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setStringListVal( + ValueProto.StringList.newBuilder() + .addAllVal( + ((GenericData.Array) featureValue) + .stream().map(Utf8::toString).collect(Collectors.toList())) + .build()) + .build(); + break; + case INT64_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setInt64ListVal( + ValueProto.Int64List.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case INT32_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setInt32ListVal( + ValueProto.Int32List.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case FLOAT_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setFloatListVal( + ValueProto.FloatList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case DOUBLE_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setDoubleListVal( + ValueProto.DoubleList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case BOOL_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setBoolListVal( + ValueProto.BoolList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case BYTES_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setBytesListVal( + ValueProto.BytesList.newBuilder() + .addAllVal( + ((GenericData.Array) featureValue) + .stream() + .map(byteBuffer -> ByteString.copyFrom(byteBuffer.array())) + .collect(Collectors.toList())) + .build()) + .build(); + break; + default: + throw new RuntimeException("FeatureType is not supported"); + } + } catch (ClassCastException e) { + // Feature type has changed + finalValue = ValueProto.Value.newBuilder().build(); + } + + return finalValue; + } + + @Override + public ServingAPIProto.FeatureReferenceV2 getFeatureReference() { + return this.featureReference; + } + + @Override + public Timestamp getEventTimestamp() { + return this.eventTimestamp; + } +} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java b/storage/api/src/main/java/feast/storage/api/retriever/Feature.java index c6cee08..92ae1f3 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/Feature.java @@ -16,33 +16,37 @@ */ package feast.storage.api.retriever; -import com.google.auto.value.AutoValue; import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; - -@AutoValue -public abstract class Feature { - - public abstract FeatureReferenceV2 getFeatureReference(); - - public abstract Value getFeatureValue(); - - public abstract Timestamp getEventTimestamp(); - - public static Builder builder() { - return new AutoValue_Feature.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setFeatureReference(FeatureReferenceV2 featureReference); - - public abstract Builder setFeatureValue(Value featureValue); - - public abstract Builder setEventTimestamp(Timestamp eventTimestamp); - - public abstract Feature build(); - } +import java.util.HashMap; + +public interface Feature { + + HashMap TYPE_TO_VAL_CASE = + new HashMap() { + { + put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); + put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); + put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); + put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); + put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); + put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); + put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); + put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); + put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); + put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); + put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); + put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); + put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); + put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); + } + }; + + Value getFeatureValue(ValueProto.ValueType.Enum valueType); + + FeatureReferenceV2 getFeatureReference(); + + Timestamp getEventTimestamp(); } diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java index 9be66a7..a49ab3f 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java @@ -33,11 +33,13 @@ public interface OnlineRetrieverV2 { * @param project name of project to request features from. * @param entityRows list of entity rows to request features for. * @param featureReferences specifies the FeatureTable to retrieve data from + * @param entityNames name of entities * @return list of {@link Feature}s corresponding to data retrieved for each entity row from * FeatureTable specified in FeatureTable request. */ List> getOnlineFeatures( String project, List entityRows, - List featureReferences); + List featureReferences, + List entityNames); } diff --git a/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java b/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java new file mode 100644 index 0000000..09f6b75 --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.retriever; + +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; + +public class ProtoFeature implements Feature { + private final ServingAPIProto.FeatureReferenceV2 featureReference; + + private final Timestamp eventTimestamp; + + private final ValueProto.Value featureValue; + + public ProtoFeature( + ServingAPIProto.FeatureReferenceV2 featureReference, + Timestamp eventTimestamp, + ValueProto.Value featureValue) { + this.featureReference = featureReference; + this.eventTimestamp = eventTimestamp; + this.featureValue = featureValue; + } + + /** + * Returns Feast valueType if type matches, otherwise null. + * + * @param valueType Feast valueType of feature as specified in FeatureSpec + * @return ValueProto.Value representation of feature + */ + @Override + public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) { + if (TYPE_TO_VAL_CASE.get(valueType) != this.featureValue.getValCase()) { + return null; + } + + return this.featureValue; + } + + @Override + public ServingAPIProto.FeatureReferenceV2 getFeatureReference() { + return this.featureReference; + } + + @Override + public Timestamp getEventTimestamp() { + return this.eventTimestamp; + } +} diff --git a/storage/connectors/bigtable/pom.xml b/storage/connectors/bigtable/pom.xml new file mode 100644 index 0000000..81cd450 --- /dev/null +++ b/storage/connectors/bigtable/pom.xml @@ -0,0 +1,45 @@ + + + + dev.feast + feast-storage-connectors + ${revision} + + + 4.0.0 + feast-storage-connector-bigtable + + + 11 + 11 + + + + + com.google.cloud + google-cloud-bigtable + 1.21.2 + + + + + org.apache.avro + avro + 1.10.2 + + + + dev.feast + feast-storage-connector-sstable + ${project.version} + + + + com.google.guava + guava + + + + \ No newline at end of file diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java new file mode 100644 index 0000000..6e67782 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -0,0 +1,211 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.storage.api.retriever.AvroFeature; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; +import java.io.IOException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; + +public class BigTableOnlineRetriever implements SSTableOnlineRetriever { + + private BigtableDataClient client; + private BigTableSchemaRegistry schemaRegistry; + + public BigTableOnlineRetriever(BigtableDataClient client) { + this.client = client; + this.schemaRegistry = new BigTableSchemaRegistry(client); + } + + /** + * Generate BigTable key in the form of entity values joined by #. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return BigTable key for retrieval + */ + @Override + public ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) { + return ByteString.copyFrom( + entityNames.stream() + .map(entity -> entityRow.getFieldsMap().get(entity)) + .map(this::valueToString) + .collect(Collectors.joining("#")) + .getBytes()); + } + + /** + * Converts rowCell feature value into @NativeFeature type. + * + * @param tableName Name of BigTable table + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + @Override + public List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences) { + + BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); + + return rowKeys.stream() + .map( + rowKey -> { + if (!rows.containsKey(rowKey)) { + return Collections.emptyList(); + } else { + Row row = rows.get(rowKey); + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .map(cf -> row.getCells(cf, "")) + .filter(ls -> !ls.isEmpty()) + .flatMap( + rowCells -> { + RowCell rowCell = rowCells.get(0); // Latest cell + String family = rowCell.getFamily(); + ByteString value = rowCell.getValue(); + + List features; + List localFeatureReferences = + featureReferences.stream() + .filter( + featureReference -> + featureReference.getFeatureTable().equals(family)) + .collect(Collectors.toList()); + + try { + features = + decodeFeatures( + tableName, + value, + localFeatureReferences, + reusedDecoder, + rowCell.getTimestamp()); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from BigTable"); + } + + return features.stream(); + }) + .collect(Collectors.toList()); + } + }) + .collect(Collectors.toList()); + } + + /** + * Retrieve rows for each row entity key by generating BigTable rowQuery with filters based on + * column families. + * + * @param tableName Name of BigTable table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + Query rowQuery = Query.create(tableName); + Filters.InterleaveFilter familyFilter = Filters.FILTERS.interleave(); + columnFamilies.forEach(cf -> familyFilter.filter(Filters.FILTERS.family().exactMatch(cf))); + + for (ByteString rowKey : rowKeys) { + rowQuery.rowKey(rowKey); + } + + return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false) + .collect(Collectors.toMap(Row::getKey, Function.identity())); + } + + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is + * returned when null is retrieved from BigTable RowCell. + * + * @param tableName Name of BigTable table + * @param value Value of BigTable cell where first 4 bytes represent the schema reference and + * remaining bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in BigTable RowCell + * @throws IOException + */ + private List decodeFeatures( + String tableName, + ByteString value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + ByteString schemaReferenceBytes = value.substring(0, 4); + byte[] featureValueBytes = value.substring(4).toByteArray(); + + BigTableSchemaRegistry.SchemaReference schemaReference = + new BigTableSchemaRegistry.SchemaReference(tableName, schemaReferenceBytes); + + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + + reusedDecoder = DecoderFactory.get().binaryDecoder(featureValueBytes, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java new file mode 100644 index 0000000..64989f0 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Iterables; +import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutionException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; + +public class BigTableSchemaRegistry { + private final BigtableDataClient client; + private final LoadingCache> cache; + + private static String COLUMN_FAMILY = "metadata"; + private static String QUALIFIER = "avro"; + private static String KEY_PREFIX = "schema#"; + + public static class SchemaReference { + private final String tableName; + private final ByteString schemaHash; + + public SchemaReference(String tableName, ByteString schemaHash) { + this.tableName = tableName; + this.schemaHash = schemaHash; + } + + public String getTableName() { + return tableName; + } + + public ByteString getSchemaHash() { + return schemaHash; + } + + @Override + public int hashCode() { + int result = tableName.hashCode(); + result = 31 * result + schemaHash.hashCode(); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SchemaReference that = (SchemaReference) o; + + if (!tableName.equals(that.tableName)) return false; + return schemaHash.equals(that.schemaHash); + } + } + + public BigTableSchemaRegistry(BigtableDataClient client) { + this.client = client; + + CacheLoader> schemaCacheLoader = + CacheLoader.from(this::loadReader); + + cache = CacheBuilder.newBuilder().build(schemaCacheLoader); + } + + public GenericDatumReader getReader(SchemaReference reference) { + GenericDatumReader reader; + try { + reader = this.cache.get(reference); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { + throw new RuntimeException(String.format("Unable to find Schema"), e); + } + return reader; + } + + private GenericDatumReader loadReader(SchemaReference reference) { + Row row = + client.readRow( + reference.getTableName(), + ByteString.copyFrom(KEY_PREFIX.getBytes()).concat(reference.getSchemaHash()), + Filters.FILTERS.family().exactMatch(COLUMN_FAMILY)); + RowCell last = Iterables.getLast(row.getCells(COLUMN_FAMILY, QUALIFIER)); + + Schema schema = new Schema.Parser().parse(last.getValue().toStringUtf8()); + return new GenericDatumReader<>(schema); + } +} diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java new file mode 100644 index 0000000..11a0445 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +public class BigTableStoreConfig { + private final String projectId; + private final String instanceId; + private final String appProfileId; + + public BigTableStoreConfig(String projectId, String instanceId, String appProfileId) { + this.projectId = projectId; + this.instanceId = instanceId; + this.appProfileId = appProfileId; + } + + public String getProjectId() { + return this.projectId; + } + + public String getInstanceId() { + return this.instanceId; + } + + public String getAppProfileId() { + return this.appProfileId; + } +} diff --git a/storage/connectors/cassandra/pom.xml b/storage/connectors/cassandra/pom.xml new file mode 100644 index 0000000..32e4d73 --- /dev/null +++ b/storage/connectors/cassandra/pom.xml @@ -0,0 +1,45 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-cassandra + + + 11 + 11 + + + + + org.apache.avro + avro + 1.10.2 + + + + dev.feast + feast-storage-connector-sstable + ${project.version} + + + + com.datastax.oss + java-driver-core + 4.11.0 + + + + com.datastax.oss + java-driver-query-builder + 4.11.0 + + + + \ No newline at end of file diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java new file mode 100644 index 0000000..9b9de7b --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -0,0 +1,271 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.AsyncPagingIterable; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.storage.api.retriever.AvroFeature; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DecoderFactory; + +public class CassandraOnlineRetriever implements SSTableOnlineRetriever { + + private final CqlSession session; + private final CassandraSchemaRegistry schemaRegistry; + + private static final String ENTITY_KEY = "key"; + private static final String SCHEMA_REF_SUFFIX = "__schema_ref"; + private static final String EVENT_TIMESTAMP_SUFFIX = "__event_timestamp"; + private static final int MAX_TABLE_NAME_LENGTH = 48; + + public CassandraOnlineRetriever(CqlSession session) { + this.session = session; + this.schemaRegistry = new CassandraSchemaRegistry(session); + } + + /** + * Generate Cassandra key in the form of entity values joined by #. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return Cassandra key for retrieval + */ + @Override + public ByteBuffer convertEntityValueToKey(EntityRow entityRow, List entityNames) { + return ByteBuffer.wrap( + entityNames.stream() + .map(entity -> entityRow.getFieldsMap().get(entity)) + .map(this::valueToString) + .collect(Collectors.joining("#")) + .getBytes()); + } + + /** + * Generate Cassandra table name, with limit of 48 characters. + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Cassandra table name for retrieval + */ + @Override + public String getSSTable(String project, List entityNames) { + String tableName = String.format("%s__%s", project, String.join("__", entityNames)); + return trimAndHash(tableName, MAX_TABLE_NAME_LENGTH); + } + + /** + * Converts Cassandra rows into @NativeFeature type. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + @Override + public List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences) { + + BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); + + return rowKeys.stream() + .map( + rowKey -> { + if (!rows.containsKey(rowKey)) { + return Collections.emptyList(); + } else { + Row row = rows.get(rowKey); + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .flatMap( + featureTableColumn -> { + ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); + ByteBuffer schemaRefKey = + row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); + + // Prevent retrieval of features from incorrect FeatureTable + List localFeatureReferences = + featureReferences.stream() + .filter( + featureReference -> + featureReference + .getFeatureTable() + .equals(featureTableColumn)) + .collect(Collectors.toList()); + + List features; + try { + features = + decodeFeatures( + schemaRefKey, + featureValues, + localFeatureReferences, + reusedDecoder, + row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from Cassandra"); + } + + return features.stream(); + }) + .collect(Collectors.toList()); + } + }) + .collect(Collectors.toList()); + } + + /** + * Retrieve rows for each row entity key by generating Cassandra Query with filters based on + * columns. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + List schemaRefColumns = + columnFamilies.stream().map(c -> c + SCHEMA_REF_SUFFIX).collect(Collectors.toList()); + Select query = + QueryBuilder.selectFrom(tableName) + .columns(columnFamilies) + .columns(schemaRefColumns) + .column(ENTITY_KEY); + for (String columnFamily : columnFamilies) { + query = query.writeTime(columnFamily).as(columnFamily + EVENT_TIMESTAMP_SUFFIX); + } + query = query.whereColumn(ENTITY_KEY).isEqualTo(QueryBuilder.bindMarker()); + + PreparedStatement preparedStatement = session.prepare(query.build()); + + List> completableAsyncResultSets = + rowKeys.stream() + .map(preparedStatement::bind) + .map(session::executeAsync) + .map(CompletionStage::toCompletableFuture) + .collect(Collectors.toList()); + + CompletableFuture allResultComputed = + CompletableFuture.allOf(completableAsyncResultSets.toArray(new CompletableFuture[0])); + + Map resultMap; + try { + resultMap = + allResultComputed + .thenApply( + v -> + completableAsyncResultSets.stream() + .map(CompletableFuture::join) + .filter(result -> result.remaining() != 0) + .map(AsyncPagingIterable::one) + .filter(Objects::nonNull) + .collect( + Collectors.toMap( + (Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity()))) + .get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e.getMessage()); + } + + return resultMap; + } + + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. + * + * @param schemaRefKey Schema reference key + * @param value Value of Cassandra cell where bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in Cassandra cell + * @throws IOException + */ + private List decodeFeatures( + ByteBuffer schemaRefKey, + ByteBuffer value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + + if (value == null || schemaRefKey == null) { + return Collections.emptyList(); + } + + CassandraSchemaRegistry.SchemaReference schemaReference = + new CassandraSchemaRegistry.SchemaReference(schemaRefKey); + + // Convert ByteBuffer to ByteArray + byte[] bytesArray = new byte[value.remaining()]; + value.get(bytesArray, 0, bytesArray.length); + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + reusedDecoder = DecoderFactory.get().binaryDecoder(bytesArray, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java new file mode 100644 index 0000000..7915b37 --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; + +public class CassandraSchemaRegistry { + private final CqlSession session; + private final PreparedStatement preparedStatement; + private final LoadingCache> cache; + + private static String SCHEMA_REF_TABLE = "feast_schema_reference"; + private static String SCHEMA_REF_COLUMN = "schema_ref"; + private static String SCHEMA_COLUMN = "avro_schema"; + + public static class SchemaReference { + private final ByteBuffer schemaHash; + + public SchemaReference(ByteBuffer schemaHash) { + this.schemaHash = schemaHash; + } + + public ByteBuffer getSchemaHash() { + return schemaHash; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SchemaReference that = (SchemaReference) o; + return Objects.equals(schemaHash, that.schemaHash); + } + + @Override + public int hashCode() { + return Objects.hash(schemaHash); + } + } + + public CassandraSchemaRegistry(CqlSession session) { + this.session = session; + String tableName = String.format("\"%s\"", SCHEMA_REF_TABLE); + Select query = + QueryBuilder.selectFrom(tableName) + .column(SCHEMA_COLUMN) + .whereColumn(SCHEMA_REF_COLUMN) + .isEqualTo(QueryBuilder.bindMarker()); + this.preparedStatement = session.prepare(query.build()); + + CacheLoader> schemaCacheLoader = + CacheLoader.from(this::loadReader); + + cache = CacheBuilder.newBuilder().build(schemaCacheLoader); + } + + public GenericDatumReader getReader(SchemaReference reference) { + GenericDatumReader reader; + try { + reader = this.cache.get(reference); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { + throw new RuntimeException("Unable to find Schema"); + } + return reader; + } + + private GenericDatumReader loadReader(SchemaReference reference) { + BoundStatement statement = preparedStatement.bind(reference.getSchemaHash()); + + Row row = session.execute(statement).one(); + + Schema schema = + new Schema.Parser() + .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); + return new GenericDatumReader<>(schema); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java new file mode 100644 index 0000000..3ee25df --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +public class CassandraStoreConfig { + + private final String connectionString; + private final String dataCenter; + private final String keySpace; + + public CassandraStoreConfig(String connectionString, String dataCenter, String keySpace) { + this.connectionString = connectionString; + this.dataCenter = dataCenter; + this.keySpace = keySpace; + } + + public String getConnectionString() { + return this.connectionString; + } + + public String getDataCenter() { + return this.dataCenter; + } + + public String getKeySpace() { + return this.keySpace; + } +} diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index 4969364..1c4b75a 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -16,6 +16,9 @@ redis + bigtable + cassandra + sstable diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index 44f74d3..e24b3bd 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -22,6 +22,7 @@ import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.ProtoFeature; import io.lettuce.core.KeyValue; import java.nio.charset.StandardCharsets; import java.util.*; @@ -33,17 +34,17 @@ public class RedisHashDecoder { * * @param redisHashValues retrieved Redis Hash values based on EntityRows * @param byteToFeatureReferenceMap map to decode bytes back to FeatureReference + * @param timestampPrefix timestamp prefix * @return List of {@link Feature} - * @throws InvalidProtocolBufferException + * @throws InvalidProtocolBufferException if a protocol buffer exception occurs */ public static List retrieveFeature( List> redisHashValues, - Map byteToFeatureReferenceMap, + Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { List allFeatures = new ArrayList<>(); - Map allFeaturesBuilderMap = - new HashMap<>(); + HashMap featureMap = new HashMap<>(); Map featureTableTimestampMap = new HashMap<>(); for (KeyValue entity : redisHashValues) { @@ -57,23 +58,22 @@ public static List retrieveFeature( featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); } else { ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK.toString()); + byteToFeatureReferenceMap.get(redisValueK); ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); - Feature.Builder featureBuilder = - Feature.builder().setFeatureReference(featureReference).setFeatureValue(featureValue); - allFeaturesBuilderMap.put(featureReference, featureBuilder); + featureMap.put(featureReference, featureValue); } } } // Add timestamp to features - for (Map.Entry entry : - allFeaturesBuilderMap.entrySet()) { + for (Map.Entry entry : + featureMap.entrySet()) { String timestampRedisHashKeyStr = timestampPrefix + ":" + entry.getKey().getFeatureTable(); Timestamp curFeatureTimestamp = featureTableTimestampMap.get(timestampRedisHashKeyStr); - Feature curFeature = entry.getValue().setEventTimestamp(curFeatureTimestamp).build(); + ProtoFeature curFeature = + new ProtoFeature(entry.getKey(), curFeatureTimestamp, entry.getValue()); allFeatures.add(curFeature); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java new file mode 100644 index 0000000..6220dd2 --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import feast.proto.storage.RedisProto; + +@FunctionalInterface +public interface EntityKeySerializer { + byte[] serialize(final RedisProto.RedisKeyV2 entityKey); +} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java new file mode 100644 index 0000000..922a09d --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import com.google.common.primitives.UnsignedBytes; +import com.google.protobuf.ProtocolStringList; +import feast.proto.storage.RedisProto; +import feast.proto.types.ValueProto; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; + +// This is derived from +// https://github.com/feast-dev/feast/blob/b1ccf8dd1535f721aee8bea937ee38feff80bec5/sdk/python/feast/infra/key_encoding_utils.py#L22 +// and must be kept up to date with any changes in that logic. +public class EntityKeySerializerV2 implements EntityKeySerializer { + + @Override + public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { + final ProtocolStringList joinKeys = entityKey.getEntityNamesList(); + final List values = entityKey.getEntityValuesList(); + + assert joinKeys.size() == values.size(); + + final List buffer = new ArrayList<>(); + + final List> tuples = new ArrayList<>(joinKeys.size()); + for (int i = 0; i < joinKeys.size(); i++) { + tuples.add(Pair.of(joinKeys.get(i), values.get(i))); + } + tuples.sort(Comparator.comparing(Pair::getLeft)); + + ByteBuffer stringBytes = ByteBuffer.allocate(Integer.BYTES); + stringBytes.order(ByteOrder.LITTLE_ENDIAN); + stringBytes.putInt(ValueProto.ValueType.Enum.STRING.getNumber()); + + for (Pair pair : tuples) { + for (final byte b : stringBytes.array()) { + buffer.add(b); + } + for (final byte b : pair.getLeft().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + } + + for (Pair pair : tuples) { + final ValueProto.Value val = pair.getRight(); + switch (val.getValCase()) { + case STRING_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.STRING.getNumber())); + buffer.add( + UnsignedBytes.checkedCast( + val.getStringVal().getBytes(StandardCharsets.UTF_8).length)); + for (final byte b : val.getStringVal().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + break; + case BYTES_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.BYTES.getNumber())); + for (final byte b : val.getBytesVal().toByteArray()) { + buffer.add(b); + } + break; + case INT32_VAL: + ByteBuffer int32ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int32ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int32ByteBuffer.putInt(ValueProto.ValueType.Enum.INT32.getNumber()); + int32ByteBuffer.putInt(Integer.BYTES); + int32ByteBuffer.putInt(val.getInt32Val()); + for (final byte b : int32ByteBuffer.array()) { + buffer.add(b); + } + break; + case INT64_VAL: + ByteBuffer int64ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int64ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int64ByteBuffer.putInt(ValueProto.ValueType.Enum.INT64.getNumber()); + int64ByteBuffer.putInt(Integer.BYTES); + /* This is super dumb - but in https://github.com/feast-dev/feast/blob/dcae1606f53028ce5413567fb8b66f92cfef0f8e/sdk/python/feast/infra/key_encoding_utils.py#L9 + we use `struct.pack("> getOnlineFeatures( String project, List entityRows, - List featureReferences) { + List featureReferences, + List entityNames) { List redisKeys = RedisKeyGenerator.buildRedisKeys(project, entityRows); List> features = getFeaturesFromRedis(redisKeys, featureReferences); @@ -57,11 +63,11 @@ private List> getFeaturesFromRedis( List featureReferences) { List> features = new ArrayList<>(); // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); + Map byteToFeatureReferenceMap = new HashMap<>(); // Serialize using proto List binaryRedisKeys = - redisKeys.stream().map(redisKey -> redisKey.toByteArray()).collect(Collectors.toList()); + redisKeys.stream().map(this.keySerializer::serialize).collect(Collectors.toList()); List featureReferenceWithTsByteList = new ArrayList<>(); featureReferences.stream() @@ -72,7 +78,7 @@ private List> getFeaturesFromRedis( byte[] featureReferenceBytes = RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); + byteToFeatureReferenceMap.put(featureReferenceBytes, featureReference); // eg. <_ts:featuretable_name> byte[] featureTableTsBytes = @@ -96,6 +102,7 @@ private List> getFeaturesFromRedis( future -> { try { List> redisValuesList = future.get(); + List curRedisKeyFeatures = RedisHashDecoder.retrieveFeature( redisValuesList, byteToFeatureReferenceMap, timestampPrefix); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java index faa8e96..e1699bb 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java @@ -52,6 +52,11 @@ public static RedisClientAdapter create(RedisStoreConfig config) { if (config.getSsl()) { uri.setSsl(true); } + + if (!config.getPassword().isEmpty()) { + uri.setPassword(config.getPassword()); + } + StatefulRedisConnection connection = io.lettuce.core.RedisClient.create(uri).connect(new ByteArrayCodec()); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java index 5395b72..e5bad29 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java @@ -22,8 +22,17 @@ import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.resource.DnsResolvers; +import io.lettuce.core.resource.MappingSocketAddressResolver; +import io.lettuce.core.resource.NettyCustomizer; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.epoll.EpollChannelOption; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class RedisClusterClient implements RedisClientAdapter { @@ -62,18 +71,81 @@ private RedisClusterClient(Builder builder) { this.asyncCommands.setAutoFlushCommands(false); } + public static String getAddressString(String host) { + try { + return InetAddress.getByName(host).getHostAddress(); + } catch (UnknownHostException e) { + throw new RuntimeException(String.format("getAllByName() failed: %s", e.getMessage())); + } + } + + public static MappingSocketAddressResolver customSocketAddressResolver( + RedisClusterStoreConfig config) { + + List configuredHosts = + Arrays.stream(config.getConnectionString().split(",")) + .map( + hostPort -> { + return hostPort.trim().split(":")[0]; + }) + .collect(Collectors.toList()); + + Map mapAddressHost = + configuredHosts.stream() + .collect( + Collectors.toMap(host -> ((String) getAddressString(host)), host -> (String) host)); + + return MappingSocketAddressResolver.create( + DnsResolvers.UNRESOLVED, + hostAndPort -> + mapAddressHost.keySet().stream().anyMatch(i -> i.equals(hostAndPort.getHostText())) + ? hostAndPort.of( + mapAddressHost.get(hostAndPort.getHostText()), hostAndPort.getPort()) + : hostAndPort); + } + + public static ClientResources customClientResources(RedisClusterStoreConfig config) { + ClientResources clientResources = + ClientResources.builder() + .nettyCustomizer( + new NettyCustomizer() { + @Override + public void afterBootstrapInitialized(Bootstrap bootstrap) { + bootstrap.option(EpollChannelOption.TCP_KEEPIDLE, 15); + bootstrap.option(EpollChannelOption.TCP_KEEPINTVL, 5); + bootstrap.option(EpollChannelOption.TCP_KEEPCNT, 3); + // Socket Timeout (milliseconds) + bootstrap.option(EpollChannelOption.TCP_USER_TIMEOUT, 60000); + } + }) + .socketAddressResolver(customSocketAddressResolver(config)) + .build(); + return clientResources; + } + public static RedisClientAdapter create(RedisClusterStoreConfig config) { + List redisURIList = Arrays.stream(config.getConnectionString().split(",")) .map( hostPort -> { String[] hostPortSplit = hostPort.trim().split(":"); - return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); + RedisURI redisURI = + RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); + if (!config.getPassword().isEmpty()) { + redisURI.setPassword(config.getPassword()); + } + if (config.getSsl()) { + redisURI.setSsl(true); + } + return redisURI; }) .collect(Collectors.toList()); io.lettuce.core.cluster.RedisClusterClient client = - io.lettuce.core.cluster.RedisClusterClient.create(redisURIList); + io.lettuce.core.cluster.RedisClusterClient.create( + customClientResources(config), redisURIList); + client.setOptions( ClusterClientOptions.builder() .socketOptions(SocketOptions.builder().keepAlive(true).tcpNoDelay(true).build()) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java index c179ffe..a7278bd 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java @@ -23,11 +23,16 @@ public class RedisClusterStoreConfig { private final String connectionString; private final ReadFrom readFrom; private final Duration timeout; + private final String password; + private final Boolean ssl; - public RedisClusterStoreConfig(String connectionString, ReadFrom readFrom, Duration timeout) { + public RedisClusterStoreConfig( + String connectionString, ReadFrom readFrom, Duration timeout, Boolean ssl, String password) { this.connectionString = connectionString; this.readFrom = readFrom; this.timeout = timeout; + this.password = password; + this.ssl = ssl; } public String getConnectionString() { @@ -41,4 +46,12 @@ public ReadFrom getReadFrom() { public Duration getTimeout() { return this.timeout; } + + public String getPassword() { + return this.password; + } + + public Boolean getSsl() { + return this.ssl; + } } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java index 5e4560a..3045235 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java @@ -20,11 +20,13 @@ public class RedisStoreConfig { private final String host; private final Integer port; private final Boolean ssl; + private final String password; - public RedisStoreConfig(String host, Integer port, Boolean ssl) { + public RedisStoreConfig(String host, Integer port, Boolean ssl, String password) { this.host = host; this.port = port; this.ssl = ssl; + this.password = password; } public String getHost() { @@ -38,4 +40,8 @@ public Integer getPort() { public Boolean getSsl() { return this.ssl; } + + public String getPassword() { + return this.password; + } } diff --git a/storage/connectors/sstable/pom.xml b/storage/connectors/sstable/pom.xml new file mode 100644 index 0000000..8c7a271 --- /dev/null +++ b/storage/connectors/sstable/pom.xml @@ -0,0 +1,19 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-sstable + + + 11 + 11 + + + \ No newline at end of file diff --git a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java new file mode 100644 index 0000000..c86923a --- /dev/null +++ b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.sstable.retriever; + +import com.google.common.hash.Hashing; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.proto.types.ValueProto; +import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.OnlineRetrieverV2; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @param Decoded value type of the partition key + * @param Type of the SSTable row + */ +public interface SSTableOnlineRetriever extends OnlineRetrieverV2 { + + int MAX_TABLE_NAME_LENGTH = 50; + + @Override + default List> getOnlineFeatures( + String project, + List entityRows, + List featureReferences, + List entityNames) { + + List columnFamilies = getSSTableColumns(featureReferences); + String tableName = getSSTable(project, entityNames); + + List rowKeys = + entityRows.stream() + .map(row -> convertEntityValueToKey(row, entityNames)) + .collect(Collectors.toList()); + + Map rowsFromSSTable = getFeaturesFromSSTable(tableName, rowKeys, columnFamilies); + + return convertRowToFeature(tableName, rowKeys, rowsFromSSTable, featureReferences); + } + + /** + * Generate SSTable key. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return SSTable key for retrieval + */ + K convertEntityValueToKey(EntityRow entityRow, List entityNames); + + /** + * Converts SSTable rows into @NativeFeature type. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences); + + /** + * Retrieve rows for each row entity key. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of column names + * @return Map of retrieved features for each rowKey + */ + Map getFeaturesFromSSTable(String tableName, List rowKeys, List columnFamilies); + + /** + * Retrieve name of SSTable corresponding to entities in retrieval call + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Name of Cassandra table + */ + default String getSSTable(String project, List entityNames) { + return trimAndHash( + String.format("%s__%s", project, String.join("__", entityNames)), MAX_TABLE_NAME_LENGTH); + } + + /** + * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, + * INT64_VAL, INT32_VAL and BYTES_VAL. + * + * @param v Entity value of Feast valueType + * @return String representation of Entity value + */ + default String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + + /** + * Retrieve SSTable columns based on Feature references. + * + * @param featureReferences List of feature references in retrieval call + * @return List of String of column names + */ + default List getSSTableColumns(List featureReferences) { + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .collect(Collectors.toList()); + } + + /** + * Trims long SSTable table names and appends hash suffix for uniqueness. + * + * @param expr Original SSTable table name + * @param maxLength Maximum length allowed for SSTable + * @return Hashed suffix SSTable table name + */ + default String trimAndHash(String expr, int maxLength) { + // Length 8 as derived from murmurhash_32 implementation + int maxPrefixLength = maxLength - 8; + String finalName = expr; + if (expr.length() > maxLength) { + String hashSuffix = + Hashing.murmur3_32().hashBytes(expr.substring(maxPrefixLength).getBytes()).toString(); + finalName = expr.substring(0, Math.min(expr.length(), maxPrefixLength)).concat(hashSuffix); + } + return finalName; + } +}