diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 7f6c3fe..c33ff8f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -42,6 +42,16 @@ jobs: integration-test: runs-on: ubuntu-latest needs: unit-test-java + services: + redis: + image: redis + ports: + - 6389:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v2 with: @@ -54,7 +64,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 348362e..417a99d 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@master with: version: '290.0.1' export_default_credentials: true diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml new file mode 100644 index 0000000..cf8527a --- /dev/null +++ b/.github/workflows/mirror.yml @@ -0,0 +1,25 @@ +name: mirror + +on: + push: + branches: + - master + tags: + - 'v*.*.*' + +jobs: + mirror: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: webfactory/ssh-agent@v0.4.1 + with: + ssh-private-key: ${{ secrets.MIRROR_SSH_KEY }} + - name: Mirror all origin branches and tags to internal repo + run: | + export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + git remote add internal ${{ secrets.INTERNAL_REPO }} + git push internal --all -f + git push internal --tags -f diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dcc377f..8899b8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -91,7 +91,7 @@ jobs: -f infra/docker/${{ matrix.component }}/Dockerfile . docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} - + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] then @@ -102,6 +102,7 @@ jobs: fi publish-java-sdk: + container: maven:3.6-jdk-11 runs-on: [ubuntu-latest] needs: get-version steps: @@ -127,8 +128,36 @@ jobs: - name: Publish java sdk env: VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} run: | - echo -n ${{ secrets.GPG_PUBLIC_KEY }} > /etc/gpg/public-key - echo -n ${{ secrets.GPG_PRIVATE_KEY }} > /etc/gpg/private-key - echo -n ${{ secrets.MAVEN_SETTINGS }} > /root/.m2/settings.xml - infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} + echo -n "$GPG_PUBLIC_KEY" > /root/public-key + echo -n "$GPG_PRIVATE_KEY" > /root/private-key + mkdir -p /root/.m2/ + echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml + infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root + + publish-helm-charts: + runs-on: ubuntu-latest + needs: get-version + env: + HELM_VERSION: v2.17.0 + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + steps: + - uses: actions/checkout@v2 + - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + with: + version: '290.0.1' + export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - run: gcloud auth configure-docker --quiet + - name: Remove previous Helm + run: sudo rm -rf $(which helm) + - name: Install Helm + run: ./infra/scripts/install-helm.sh + - name: Validate all version consistency + run: ./infra/scripts/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX + - name: Publish Helm charts + run: ./infra/scripts/push-helm-charts.sh $VERSION_WITHOUT_PREFIX \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index a908f12..df8838f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "deps/feast"] path = deps/feast url = https://github.com/feast-dev/feast - branch = v0.9-branch + branch = v0.10-branch diff --git a/.scalafmt.conf b/.scalafmt.conf deleted file mode 100644 index f3c72b8..0000000 --- a/.scalafmt.conf +++ /dev/null @@ -1,2 +0,0 @@ -align.preset = more -maxColumn = 100 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e4e04e8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,167 @@ +# Development Guide: feast-java +> The higher level [Development Guide](https://docs.feast.dev/contributing/development-guide) +> gives contributing to Feast codebase as a whole. + +### Overview +This guide is targeted at developers looking to contribute to Feast components in +the feast-java Repository: +- [Feast Core](#feast-core) +- [Feast Serving](#feast-serving) +- [Feast Java Client](#feast-java-client) + +> Don't see the Feast component that you want to contribute to here? +> Check out the [Development Guide](https://docs.feast.dev/contributing/development-guide) +> to learn how Feast components are distributed over multiple repositories. + +#### Common Setup +Common Environment Setup for all feast-java Feast components: +1. feast-java contains submodules that need to be updated: +```sh +git submodule init +git submodule update --recursive +``` +2. Ensure following development tools are installed: +- Java SE Development Kit 11, Maven 3.6, `make` + +#### Code Style +feast-java's codebase conforms to the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). + +Automatically format the code to conform the style guide by: + +```sh +# formats all code in the feast-java repository +mvn spotless:apply +``` + +> If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) +> if you'd like to use the IDE's reformat function. + +#### Project Makefile +The Project Makefile provides useful shorthands for common development tasks: + + +Run all Unit tests: +``` +make test-java +``` + +Run all Integration tests: +``` +make test-java-integration +``` + +Building Docker images for Feast Core & Feast Serving: +``` +make build-docker REGISTRY=gcr.io/kf-feast VERSION=develop +``` + + +#### IDE Setup +If you're using IntelliJ, some additional steps may be needed to make sure IntelliJ autocomplete works as expected. +Specifically, proto-generated code is not indexed by IntelliJ. To fix this, navigate to the following window in IntelliJ: +`Project Structure > Modules > datatypes-java`, and mark the following folders as `Source` directorys: +- target/generated-sources/protobuf/grpc-java +- target/generated-sources/protobuf/java +- target/generated-sources/annotations + + +## Feast Core +### Environment Setup +Setting up your development environment for Feast Core: +1. Complete the feast-java [Common Setup](#common-setup) +2. Boot up a PostgreSQL instance (version 11 and above). Example of doing so via Docker: +```sh +# spawn a PostgreSQL instance as a Docker container running in the background +docker run \ + --rm -it -d \ + --name postgres \ + -e POSTGRES_DB=postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=password \ + -p 5432:5432 postgres:12-alpine +``` + +### Configuration +Feast Core is configured using it's [application.yml](https://docs.feast.dev/reference/configuration-reference#1-feast-core-and-feast-online-serving). + +### Building and Running +1. Build / Compile Feast Core with Maven to produce an executable Feast Core JAR +```sh +mvn package -pl core --also-make -Dmaven.test.skip=true +``` + +2. Run Feast Core using the built JAR: +```sh +# where X.X.X is the version of the Feast Core JAR built +java -jar core/target/feast-core-X.X.X-exec.jar +``` + +### Unit / Integration Tests +Unit & Integration Tests can be used to verify functionality: +```sh +# run unit tests +mvn test -pl core --also-make +# run integration tests +mvn verify -pl core --also-make +``` + +## Feast Serving +### Environment Setup +Setting up your development environment for Feast Serving: +1. Complete the feast-java [Common Setup](#common-setup) +2. Boot up a Redis instance (version 5.x). Example of doing so via Docker: +```sh +docker run --name redis --rm -it -d -p 6379:6379 redis:5-alpine +``` + +> Feast Serving requires a running Feast Core instance to retrieve Feature metadata +> in order to serve features. See the [Feast Core section](#feast-core) for +> how to get a Feast Core instance running. + +### Configuration +Feast Serving is configured using it's [application.yml](https://docs.feast.dev/reference/configuration-reference#1-feast-core-and-feast-online-serving). + +### Building and Running +1. Build / Compile Feast Serving with Maven to produce an executable Feast Serving JAR +```sh +mvn package -pl serving --also-make -Dmaven.test.skip=true + +2. Run Feast Serving using the built JAR: +```sh +# where X.X.X is the version of the Feast serving JAR built +java -jar serving/target/feast-serving-X.X.X-exec.jar +``` + +### Unit / Integration Tests +Unit & Integration Tests can be used to verify functionality: +```sh +# run unit tests +mvn test -pl serving --also-make +# run integration tests +mvn verify -pl serving --also-make +``` + +## Feast Java Client +### Environment Setup +Setting up your development environment for Feast Java SDK: +1. Complete the feast-java [Common Setup](#common-setup) + +> Feast Java Client is a Java Client for retrieving Features from a running Feast Serving instance. +> See the [Feast Serving Section](#feast-serving) section for how to get a Feast Serving instance running. + +### Configuration +Feast Java Client is [configured as code](https://docs.feast.dev/v/master/reference/configuration-reference#4-feast-java-and-go-sdk) + +### Building +1. Build / Compile Feast Java Client with Maven: + +```sh +mvn package -pl sdk/java --also-make -Dmaven.test.skip=true +``` + +### Unit Tests +Unit Tests can be used to verify functionality: + +```sh +mvn package -pl sdk/java test --also-make +``` diff --git a/Makefile b/Makefile index b1f95dc..7d00913 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,10 @@ push-serving-docker: docker push $(REGISTRY)/feast-serving:$(VERSION) build-core-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . + docker build --no-cache --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . build-serving-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . + docker build --no-cache --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . # Versions diff --git a/README.md b/README.md index 52de621..98df035 100644 --- a/README.md +++ b/README.md @@ -1 +1,31 @@ -Feast Java components +# Feast Java components (deprecated) +[![complete](https://github.com/feast-dev/feast-java/actions/workflows/complete.yml/badge.svg)](https://github.com/feast-dev/feast-java/actions/workflows/complete.yml) + +### Note: This repository worked with Feast 0.9 and before. Please look at http://github.com/feast-dev/feast for the more up to date version of this repo. + +### Overview + +This repository contains the following Feast components. +* Feast Core: The central feature registry used to define and manage entities and features. +* Feast Serving: A service used to serve the latest feature values to models. +* Feast Java SDK: A client used to retrieve features from Feast Serving. +* Helm Charts: The repository also contains Helm charts to deploy Feast Core and Feast Serving into a Kubernetes cluster. + +### Architecture + +![](docs/architecture.png) + + +* Feast Core has a dependency on Postgres. +* Feast Serving has a dependency on an online store (Redis) for retrieving features. The process of ingesting data into the online store (Redis) is decoupled from the process of reading from it. Please see [Feast Spark](https://github.com/feast-dev/feast-spark) for more details about ingesting data into the online store. +* Feast Serving has a dependency on Feast Core. +* The Go and Python Clients are not a part of this repository. + +### Contributing +Guides on Contributing: +- [Contribution Process for Feast](https://docs.feast.dev/v/master/contributing/contributing) +- [Development Guide for Feast](https://docs.feast.dev/contributing/development-guide) +- [Development Guide for feast-java (this repository)](./CONTRIBUTING.md) + +### Installing using Helm +Please see the Helm charts in [charts](infra/charts). diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java index f82a804..8d49b38 100644 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -115,7 +115,7 @@ public ConsumerFactory testConsumerFactory() { /** * Truncates all tables in Database (between tests or flows). Retries on deadlock * - * @throws SQLException + * @throws SQLException when a SQL exception occurs */ public static void cleanTables() throws SQLException { Connection connection = @@ -156,7 +156,12 @@ public static void cleanTables() throws SQLException { } } - /** Used to determine SequentialFlows */ + /** + * Used to determine SequentialFlows + * + * @param testInfo test info + * @return true if test is sequential + */ public Boolean isSequentialTest(TestInfo testInfo) { try { testInfo.getTestClass().get().asSubclass(SequentialFlow.class); diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 0606c75..8a0dbb0 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -53,6 +53,28 @@ public static Triple getDefaultSubscription() { return defaultSubscription; } + public static String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + public static StoreProto.Store getDefaultStore() { return defaultStore; } @@ -227,6 +249,10 @@ public static ValueProto.Value createDoubleValue(double value) { return ValueProto.Value.newBuilder().setDoubleVal(value).build(); } + public static ValueProto.Value createInt32Value(int value) { + return ValueProto.Value.newBuilder().setInt32Val(value).build(); + } + public static ValueProto.Value createInt64Value(long value) { return ValueProto.Value.newBuilder().setInt64Val(value).build(); } @@ -247,6 +273,18 @@ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityR .build(); } + public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createCompoundEntityRow( + ImmutableMap entityNameValues, long seconds) { + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.Builder entityRow = + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(seconds)); + + entityNameValues.entrySet().stream() + .forEach(entry -> entityRow.putFields(entry.getKey(), entry.getValue())); + + return entityRow.build(); + } + public static DataSource createKinesisDataSourceSpec( String region, String streamName, String classPath, String timestampColumn) { return DataSource.newBuilder() diff --git a/common-test/src/main/java/feast/common/util/TestUtil.java b/common-test/src/main/java/feast/common/util/TestUtil.java index ee355d3..49e6cc7 100644 --- a/common-test/src/main/java/feast/common/util/TestUtil.java +++ b/common-test/src/main/java/feast/common/util/TestUtil.java @@ -44,6 +44,10 @@ public static void setupAuditLogger() { /** * Compare if two Feature Table specs are equal. Disregards order of features/entities in spec. + * + * @param spec one spec + * @param otherSpec the other spec + * @return true if specs equal */ public static boolean compareFeatureTableSpec(FeatureTableSpec spec, FeatureTableSpec otherSpec) { spec = diff --git a/common/pom.xml b/common/pom.xml index ce89e9c..99008e1 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -125,7 +125,12 @@ com.google.auth google-auth-library-oauth2-http - + + sh.ory.keto + keto-client + 0.5.7-alpha.1.pre.0 + + org.openapitools diff --git a/common/src/main/java/feast/common/auth/config/SecurityConfig.java b/common/src/main/java/feast/common/auth/config/SecurityConfig.java index aa7f8a2..94a4018 100644 --- a/common/src/main/java/feast/common/auth/config/SecurityConfig.java +++ b/common/src/main/java/feast/common/auth/config/SecurityConfig.java @@ -19,6 +19,7 @@ import feast.common.auth.authentication.DefaultJwtAuthenticationProvider; import feast.common.auth.authorization.AuthorizationProvider; import feast.common.auth.providers.http.HttpAuthorizationProvider; +import feast.common.auth.providers.keto.KetoAuthorizationProvider; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -107,12 +108,40 @@ AccessDecisionManager accessDecisionManager() { AuthorizationProvider authorizationProvider() { if (securityProperties.getAuthentication().isEnabled() && securityProperties.getAuthorization().isEnabled()) { + // Merge authentication and authorization options to create HttpAuthorizationProvider. + Map options = securityProperties.getAuthorization().getOptions(); + + options.putAll(securityProperties.getAuthentication().getOptions()); switch (securityProperties.getAuthorization().getProvider()) { case "http": - // Merge authenticatoin and authorization options to create HttpAuthorizationProvider. - Map options = securityProperties.getAuthorization().getOptions(); - options.putAll(securityProperties.getAuthentication().getOptions()); return new HttpAuthorizationProvider(options); + case "keto": + String subjectClaim = + options.get(SecurityProperties.AuthenticationProperties.SUBJECT_CLAIM); + String flavor = options.get("flavor"); + String action = options.get("action"); + String subjectPrefix = options.get("subjectPrefix"); + String resourcePrefix = options.get("resourcePrefix"); + + KetoAuthorizationProvider.Builder builder = + new KetoAuthorizationProvider.Builder(options.get("authorizationUrl")); + if (subjectClaim != null) { + builder = builder.withSubjectClaim(subjectClaim); + } + if (flavor != null) { + builder = builder.withFlavor(flavor); + } + if (action != null) { + builder = builder.withAction(action); + } + if (subjectPrefix != null) { + builder = builder.withSubjectPrefix(subjectPrefix); + } + if (resourcePrefix != null) { + builder = builder.withResourcePrefix(resourcePrefix); + } + + return builder.build(); default: throw new IllegalArgumentException( "Please configure an Authorization Provider if you have enabled authorization."); diff --git a/common/src/main/java/feast/common/auth/config/SecurityProperties.java b/common/src/main/java/feast/common/auth/config/SecurityProperties.java index 135cc4b..f48d734 100644 --- a/common/src/main/java/feast/common/auth/config/SecurityProperties.java +++ b/common/src/main/java/feast/common/auth/config/SecurityProperties.java @@ -53,7 +53,7 @@ public static class AuthorizationProperties { private boolean enabled; // Named authorization provider to use. - @OneOfStrings({"none", "http"}) + @OneOfStrings({"none", "http", "keto"}) private String provider; // K/V options to initialize the provider with diff --git a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java b/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java index 0f42325..57aafa2 100644 --- a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java +++ b/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java @@ -45,6 +45,7 @@ public class GoogleAuthCredentials extends CallCredentials { * * @param options a map of options, Required unless specified: audience - Optional, Sets the * target audience of the token obtained. + * @throws IOException if credentials are not available */ public GoogleAuthCredentials(Map options) throws IOException { String targetAudience = options.getOrDefault("audience", "https://localhost"); diff --git a/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java b/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java new file mode 100644 index 0000000..05fcd3c --- /dev/null +++ b/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.common.auth.providers.keto; + +import feast.common.auth.authorization.AuthorizationProvider; +import feast.common.auth.authorization.AuthorizationResult; +import feast.common.auth.utils.AuthUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.core.Authentication; +import sh.ory.keto.ApiClient; +import sh.ory.keto.ApiException; +import sh.ory.keto.Configuration; +import sh.ory.keto.api.EnginesApi; +import sh.ory.keto.model.OryAccessControlPolicyAllowedInput; + +public class KetoAuthorizationProvider implements AuthorizationProvider { + + /** Builder for KetoAuthorizationProvider */ + public static class Builder { + private final String url; + private String subjectClaim = "email"; + private String flavor = "glob"; + private String action = "edit"; + private String subjectPrefix = ""; + private String resourcePrefix = ""; + + /** + * Initialized builder for Keto authorization provider. + * + * @param url Url string for Keto server. + */ + public Builder(String url) { + this.url = url; + } + + /** + * Set subject claim for authentication + * + * @param subjectClaim Subject claim. Default: email. + * @return Returns Builder + */ + public Builder withSubjectClaim(String subjectClaim) { + this.subjectClaim = subjectClaim; + return this; + } + + /** + * Set flavor for Keto authorization. One of [exact, glob regex] + * + * @param flavor Keto authorization flavor. Default: glob. + * @return Returns Builder + */ + public Builder withFlavor(String flavor) { + this.flavor = flavor; + return this; + } + + /** + * Set action that corresponds to the permission to edit a Feast project resource. + * + * @param action Keto action. Default: edit. + * @return Returns Builder + */ + public Builder withAction(String action) { + this.action = action; + return this; + } + + /** + * If set, The subject will be prefixed before sending the request to Keto. Example: + * users:someuser@email.com + * + * @param prefix Subject prefix. Default: Empty string. + * @return Returns Builder + */ + public Builder withSubjectPrefix(String prefix) { + this.subjectPrefix = prefix; + return this; + } + + /** + * If set, The resource will be prefixed before sending the request to Keto. Example: + * projects:somefeastproject + * + * @param prefix Resource prefix. Default: Empty string. + * @return Returns Builder + */ + public Builder withResourcePrefix(String prefix) { + this.resourcePrefix = prefix; + return this; + } + + /** + * Build KetoAuthorizationProvider + * + * @return Returns KetoAuthorizationProvider + */ + public KetoAuthorizationProvider build() { + return new KetoAuthorizationProvider(this); + } + } + + private static final Logger log = LoggerFactory.getLogger(KetoAuthorizationProvider.class); + + private final EnginesApi apiInstance; + private final String subjectClaim; + private final String flavor; + private final String action; + private final String subjectPrefix; + private final String resourcePrefix; + + private KetoAuthorizationProvider(Builder builder) { + ApiClient defaultClient = Configuration.getDefaultApiClient(); + defaultClient.setBasePath(builder.url); + apiInstance = new EnginesApi(defaultClient); + subjectClaim = builder.subjectClaim; + flavor = builder.flavor; + action = builder.action; + subjectPrefix = builder.subjectPrefix; + resourcePrefix = builder.resourcePrefix; + } + + @Override + public AuthorizationResult checkAccessToProject(String projectId, Authentication authentication) { + String subject = AuthUtils.getSubjectFromAuth(authentication, subjectClaim); + OryAccessControlPolicyAllowedInput body = new OryAccessControlPolicyAllowedInput(); + body.setAction(action); + body.setSubject(String.format("%s%s", subjectPrefix, subject)); + body.setResource(String.format("%s%s", resourcePrefix, projectId)); + try { + sh.ory.keto.model.AuthorizationResult authResult = + apiInstance.doOryAccessControlPoliciesAllow(flavor, body); + if (authResult == null) { + throw new RuntimeException( + String.format( + "Empty response returned for access to project %s for subject %s", + projectId, subject)); + } + if (authResult.getAllowed()) { + return AuthorizationResult.success(); + } + } catch (ApiException e) { + log.error("API exception has occurred during authorization: {}", e.getMessage(), e); + } + + return AuthorizationResult.failed( + String.format("Access denied to project %s for subject %s", projectId, subject)); + } +} diff --git a/common/src/main/java/feast/common/logging/AuditLogger.java b/common/src/main/java/feast/common/logging/AuditLogger.java index 0b9901e..5f70fbf 100644 --- a/common/src/main/java/feast/common/logging/AuditLogger.java +++ b/common/src/main/java/feast/common/logging/AuditLogger.java @@ -65,6 +65,7 @@ public AuditLogger(LoggingProperties loggingProperties, BuildProperties buildPro /** * Log the handling of a Protobuf message by a service call. * + * @param level log level * @param entryBuilder with all fields set except instance. */ public static void logMessage(Level level, MessageAuditLogEntry.Builder entryBuilder) { diff --git a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java index cec85b7..4fdeaee 100644 --- a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java @@ -21,10 +21,10 @@ /** ActionAuditLogEntry records an action being taken on a specific resource */ @AutoValue public abstract class ActionAuditLogEntry extends AuditLogEntry { - /** The name of the action taken on the resource. */ + /** @return The name of the action taken on the resource. */ public abstract String getAction(); - /** The target resource of which the action was taken on. */ + /** @return The target resource of which the action was taken on. */ public abstract LogResource getResource(); /** @@ -34,6 +34,7 @@ public abstract class ActionAuditLogEntry extends AuditLogEntry { * @param version The version of Feast producing this {@link AuditLogEntry}. * @param resource The target resource of which the action was taken on. * @param action The name of the action being taken on the given resource. + * @return log entry that records an action being taken on a specific resource */ public static ActionAuditLogEntry of( String component, String version, LogResource resource, String action) { diff --git a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java index 9aa8fcb..8148c47 100644 --- a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java @@ -29,15 +29,27 @@ public abstract class AuditLogEntry { public final String application = "Feast"; - /** The name of the Feast component producing this {@link AuditLogEntry} */ + /** + * The name of the Feast component producing this {@link AuditLogEntry} + * + * @return the component + */ public abstract String getComponent(); - /** The version of Feast producing this {@link AuditLogEntry} */ + /** + * The version of Feast producing this {@link AuditLogEntry} + * + * @return version + */ public abstract String getVersion(); public abstract AuditLogEntryKind getKind(); - /** Return a structured JSON representation of this {@link AuditLogEntry} */ + /** + * Return a structured JSON representation of this {@link AuditLogEntry} + * + * @return structured JSON representation + */ public String toJSON() { Gson gson = new Gson(); return gson.toJson(this); diff --git a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java index 745cc12..6e5072f 100644 --- a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java @@ -34,32 +34,35 @@ /** MessageAuditLogEntry records the handling of a Protobuf message by a service call. */ @AutoValue public abstract class MessageAuditLogEntry extends AuditLogEntry { - /** Id used to identify the service call that the log entry is recording */ + /** @return Id used to identify the service call that the log entry is recording */ public abstract UUID getId(); - /** The name of the service that was used to handle the service call. */ + /** @return The name of the service that was used to handle the service call. */ public abstract String getService(); - /** The name of the method that was used to handle the service call. */ + /** @return The name of the method that was used to handle the service call. */ public abstract String getMethod(); - /** The request Protobuf {@link Message} that was passed to the Service in the service call. */ + /** + * @return The request Protobuf {@link Message} that was passed to the Service in the service + * call. + */ public abstract Message getRequest(); /** - * The response Protobuf {@link Message} that was passed to the Service in the service call. May - * be an {@link Empty} protobuf no request could be collected due to an error. + * @return The response Protobuf {@link Message} that was passed to the Service in the service + * call. May be an {@link Empty} protobuf no request could be collected due to an error. */ public abstract Message getResponse(); /** - * The authenticated identity that was assumed during the handling of the service call. For - * example, the user id or email that identifies the user making the call. Empty if the service - * call is not authenticated. + * @return The authenticated identity that was assumed during the handling of the service call. + * For example, the user id or email that identifies the user making the call. Empty if the + * service call is not authenticated. */ public abstract String getIdentity(); - /** The result status code of the service call. */ + /** @return The result status code of the service call. */ public abstract Code getStatusCode(); @AutoValue.Builder diff --git a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java index 0f139b7..224f10e 100644 --- a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java +++ b/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java @@ -21,10 +21,10 @@ /** TransitionAuditLogEntry records a transition in state/status in a specific resource. */ @AutoValue public abstract class TransitionAuditLogEntry extends AuditLogEntry { - /** The resource which the state/status transition occured. */ + /** @return The resource which the state/status transition occured. */ public abstract LogResource getResource(); - /** The end status with the resource transition to. */ + /** @return The end status with the resource transition to. */ public abstract String getStatus(); /** @@ -35,6 +35,7 @@ public abstract class TransitionAuditLogEntry extends AuditLogEntry { * @param version The version of Feast producing this {@link AuditLogEntry}. * @param resource the resource which the transtion occured * @param status the end status which the resource transitioned to. + * @return log entry to record a transition in state/status in a specific resource */ public static TransitionAuditLogEntry of( String component, String version, LogResource resource, String status) { diff --git a/common/src/main/java/feast/common/models/FeatureTable.java b/common/src/main/java/feast/common/models/FeatureTable.java index d4712e7..88fac15 100644 --- a/common/src/main/java/feast/common/models/FeatureTable.java +++ b/common/src/main/java/feast/common/models/FeatureTable.java @@ -25,6 +25,7 @@ public class FeatureTable { * Accepts FeatureTableSpec object and returns its reference in String * "project/featuretable_name". * + * @param project project name * @param featureTableSpec {@link FeatureTableSpec} * @return String format of FeatureTableReference */ @@ -36,6 +37,7 @@ public static String getFeatureTableStringRef(String project, FeatureTableSpec f * Accepts FeatureReferenceV2 object and returns its reference in String * "project/featuretable_name". * + * @param project project name * @param featureReference {@link FeatureReferenceV2} * @return String format of FeatureTableReference */ diff --git a/common/src/main/java/feast/common/models/FeatureV2.java b/common/src/main/java/feast/common/models/FeatureV2.java index 8debca3..8420cca 100644 --- a/common/src/main/java/feast/common/models/FeatureV2.java +++ b/common/src/main/java/feast/common/models/FeatureV2.java @@ -34,4 +34,17 @@ public static String getFeatureStringRef(FeatureReferenceV2 featureReference) { } return ref; } + + /** + * Accepts either a feature reference of the form "featuretable_name:feature_name" or just a + * feature name, and returns just the feature name. For example, given either + * "driver_hourly_stats:conv_rate" or "conv_rate", "conv_rate" would be returned. + * + * @param featureReference {String} + * @return Base feature name of the feature reference + */ + public static String getFeatureName(String featureReference) { + String[] tokens = featureReference.split(":", 2); + return tokens[tokens.length - 1]; + } } diff --git a/common/src/main/java/feast/common/validators/OneOfStringValidator.java b/common/src/main/java/feast/common/validators/OneOfStringValidator.java index 42428bd..924953a 100644 --- a/common/src/main/java/feast/common/validators/OneOfStringValidator.java +++ b/common/src/main/java/feast/common/validators/OneOfStringValidator.java @@ -29,7 +29,7 @@ public class OneOfStringValidator implements ConstraintValidator[] groups() default {}; - /** An attribute payload that can be used to assign custom payload objects to a constraint. */ + /** + * @return An attribute payload that can be used to assign custom payload objects to a constraint. + */ Class[] payload() default {}; /** @return Default value that is returned if no allowed values are configured */ diff --git a/core/pom.xml b/core/pom.xml index 7a34b79..1c33e15 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -298,7 +298,6 @@ sh.ory.keto keto-client 0.4.4-alpha.1 - test com.github.tomakehurst diff --git a/core/src/main/java/feast/core/config/WebSecurityConfig.java b/core/src/main/java/feast/core/config/WebSecurityConfig.java index 0f48111..5c66730 100644 --- a/core/src/main/java/feast/core/config/WebSecurityConfig.java +++ b/core/src/main/java/feast/core/config/WebSecurityConfig.java @@ -43,7 +43,7 @@ public WebSecurityConfig(FeastProperties feastProperties) { * Allows for custom web security rules to be applied. * * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception + * @throws Exception unexpected exception */ @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index efdf0fc..648195a 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -311,14 +311,14 @@ public void applyFeatureTable( String.format( "ApplyFeatureTable: Unable to apply Feature Table due to a conflict: " + "Ensure that name is unique within Project: (name: %s, project: %s)", - projectName, tableName)); + tableName, projectName)); responseObserver.onError( Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } catch (IllegalArgumentException e) { log.error( String.format( "ApplyFeatureTable: Invalid apply Feature Table Request: (name: %s, project: %s)", - projectName, tableName)); + tableName, projectName)); responseObserver.onError( Status.INVALID_ARGUMENT .withDescription(e.getMessage()) @@ -328,7 +328,7 @@ public void applyFeatureTable( log.error( String.format( "ApplyFeatureTable: Unsupported apply Feature Table Request: (name: %s, project: %s)", - projectName, tableName)); + tableName, projectName)); responseObserver.onError( Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } catch (Exception e) { diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java index 67477da..2bfe20f 100644 --- a/core/src/main/java/feast/core/model/DataSource.java +++ b/core/src/main/java/feast/core/model/DataSource.java @@ -87,6 +87,7 @@ public DataSource(SourceType type) { * @param spec Protobuf representation of DataSource to construct from. * @throws IllegalArgumentException when provided with a invalid Protobuf spec * @throws UnsupportedOperationException if source type is unsupported. + * @return data source */ public static DataSource fromProto(DataSourceProto.DataSource spec) { DataSource source = new DataSource(spec.getType()); @@ -132,7 +133,11 @@ public static DataSource fromProto(DataSourceProto.DataSource spec) { return source; } - /** Convert this DataSource to its Protobuf representation. */ + /** + * Convert this DataSource to its Protobuf representation. + * + * @return protobuf representation + */ public DataSourceProto.DataSource toProto() { DataSourceProto.DataSource.Builder spec = DataSourceProto.DataSource.newBuilder(); spec.setType(getType()); diff --git a/core/src/main/java/feast/core/model/EntityV2.java b/core/src/main/java/feast/core/model/EntityV2.java index aeb6728..d72bef1 100644 --- a/core/src/main/java/feast/core/model/EntityV2.java +++ b/core/src/main/java/feast/core/model/EntityV2.java @@ -65,6 +65,11 @@ public EntityV2() { * *

This data model supports Scalar Entity and would allow ease of discovery of entities and * reasoning when used in association with FeatureTable. + * + * @param name name + * @param description description + * @param type type + * @param labels labels */ public EntityV2( String name, String description, ValueType.Enum type, Map labels) { diff --git a/core/src/main/java/feast/core/model/FeatureTable.java b/core/src/main/java/feast/core/model/FeatureTable.java index b442d57..9b9bdef 100644 --- a/core/src/main/java/feast/core/model/FeatureTable.java +++ b/core/src/main/java/feast/core/model/FeatureTable.java @@ -155,7 +155,9 @@ public static FeatureTable fromProto( /** * Update the FeatureTable from the given Protobuf representation. * + * @param projectName project name * @param spec the Protobuf spec to update the FeatureTable from. + * @param entityRepo repository * @throws IllegalArgumentException if the update will make prohibited changes. */ public void updateFromProto( @@ -211,7 +213,11 @@ public void updateFromProto( this.revision++; } - /** Convert this Feature Table to its Protobuf representation */ + /** + * Convert this Feature Table to its Protobuf representation + * + * @return protobuf representation + */ public FeatureTableProto.FeatureTable toProto() { // Convert field types to Protobuf compatible types Timestamp creationTime = TypeConversion.convertTimestamp(getCreated()); @@ -319,6 +325,7 @@ private Map getFeaturesRefToFeaturesMap(List featu /** * Returns a list of Features if FeatureTable's Feature contains all labels in labelsFilter * + * @param features features * @param labelsFilter contain labels that should be attached to FeatureTable's features * @return List of Features */ @@ -359,13 +366,13 @@ public void delete() { public String protoHash() { List sortedEntities = - this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toList()); - Collections.sort(sortedEntities); + this.getEntities().stream().map(EntityV2::getName).sorted().collect(Collectors.toList()); - List sortedFeatures = new ArrayList(this.getFeatures()); List sortedFeatureSpecs = - sortedFeatures.stream().map(featureV2 -> featureV2.toProto()).collect(Collectors.toList()); - sortedFeatures.sort(Comparator.comparing(FeatureV2::getName)); + this.getFeatures().stream() + .sorted(Comparator.comparing(FeatureV2::getName)) + .map(FeatureV2::toProto) + .collect(Collectors.toList()); DataSourceProto.DataSource streamSource = DataSourceProto.DataSource.getDefaultInstance(); if (getStreamSource() != null) { diff --git a/core/src/main/java/feast/core/model/FeatureV2.java b/core/src/main/java/feast/core/model/FeatureV2.java index f25e951..d0a6082 100644 --- a/core/src/main/java/feast/core/model/FeatureV2.java +++ b/core/src/main/java/feast/core/model/FeatureV2.java @@ -75,7 +75,11 @@ public static FeatureV2 fromProto(FeatureTable table, FeatureSpecV2 spec) { return new FeatureV2(table, spec.getName(), spec.getValueType(), labelsJSON); } - /** Convert this Feature to its Protobuf representation. */ + /** + * Convert this Feature to its Protobuf representation. + * + * @return protobuf representation + */ public FeatureSpecV2 toProto() { Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); return FeatureSpecV2.newBuilder() diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 4a35d3e..ff45dcd 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -17,7 +17,7 @@ package feast.core.service; import static feast.core.validators.Matchers.checkValidCharacters; -import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk; +import static feast.core.validators.Matchers.checkValidCharactersAllowDash; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.dao.EntityRepository; @@ -105,7 +105,7 @@ public GetEntityResponse getEntity(GetEntityRequest request) { projectName = Project.DEFAULT_NAME; } - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(entityName, "entity"); EntityV2 entity = entityRepository.findEntityByNameAndProject_Name(entityName, projectName); @@ -143,13 +143,13 @@ public ListFeaturesResponse listFeatures(ListFeaturesRequest.Filter filter) { List entities = filter.getEntitiesList(); Map labels = filter.getLabelsMap(); - checkValidCharactersAllowAsterisk(project, "project"); - // Autofill default project if project not specified if (project.isEmpty()) { project = Project.DEFAULT_NAME; } + checkValidCharactersAllowDash(project, "project"); + // Currently defaults to all FeatureTables List featureTables = tableRepository.findAllByProject_Name(project); @@ -200,7 +200,7 @@ public ListEntitiesResponse listEntities(ListEntitiesRequest.Filter filter) { project = Project.DEFAULT_NAME; } - checkValidCharacters(project, "project"); + checkValidCharactersAllowDash(project, "project"); List entities = entityRepository.findAllByProject_Name(project); @@ -262,6 +262,7 @@ public ListStoresResponse listStores(ListStoresRequest.Filter filter) { * * @param newEntitySpec EntitySpecV2 that will be used to create or update an Entity. * @param projectName Project namespace of Entity which is to be created/updated + * @return response of the operation */ @Transactional public ApplyEntityResponse applyEntity( @@ -271,6 +272,8 @@ public ApplyEntityResponse applyEntity( projectName = Project.DEFAULT_NAME; } + checkValidCharactersAllowDash(projectName, "project"); + // Validate incoming entity EntityValidator.validateSpec(newEntitySpec); @@ -312,6 +315,7 @@ public ApplyEntityResponse applyEntity( * Resolves the project name by returning name if given, autofilling default project otherwise. * * @param projectName name of the project to resolve. + * @return project name */ public static String resolveProjectName(String projectName) { return (projectName.isEmpty()) ? Project.DEFAULT_NAME : projectName; @@ -322,6 +326,7 @@ public static String resolveProjectName(String projectName) { * * @param updateStoreRequest containing the new store definition * @return UpdateStoreResponse containing the new store definition + * @throws InvalidProtocolBufferException if protobuf exception occurs */ @Transactional public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) @@ -368,6 +373,8 @@ public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) public ApplyFeatureTableResponse applyFeatureTable(ApplyFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); + checkValidCharactersAllowDash(projectName, "project"); + // Check that specification provided is valid FeatureTableSpec applySpec = request.getTableSpec(); FeatureTableValidator.validateSpec(applySpec); @@ -411,7 +418,7 @@ public ListFeatureTablesResponse listFeatureTables(ListFeatureTablesRequest.Filt String projectName = resolveProjectName(filter.getProject()); Map labelsFilter = filter.getLabelsMap(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); List matchingTables = tableRepository.findAllByProject_Name(projectName); @@ -444,7 +451,7 @@ public GetFeatureTableResponse getFeatureTable(GetFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); String featureTableName = request.getName(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(featureTableName, "featureTable"); Optional retrieveTable = @@ -474,7 +481,7 @@ public void deleteFeatureTable(DeleteFeatureTableRequest request) { String projectName = resolveProjectName(request.getProject()); String featureTableName = request.getName(); - checkValidCharacters(projectName, "project"); + checkValidCharactersAllowDash(projectName, "project"); checkValidCharacters(featureTableName, "featureTable"); Optional existingTable = diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index bbdfa94..d2a2d0a 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -79,7 +79,7 @@ public static Map convertJsonStringToEnumMap(String jsonString) { /** * Marshals a given map into its corresponding json string * - * @param map + * @param map map to be converted * @return json string corresponding to given map */ public static String convertMapToJsonString(Map map) { @@ -89,7 +89,7 @@ public static String convertMapToJsonString(Map map) { /** * Marshals a given Enum map into its corresponding json string * - * @param map + * @param map map to be converted * @return json string corresponding to given Enum map */ public static String convertEnumMapToJsonString(Map map) { diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java index f36e360..548d18f 100644 --- a/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ b/core/src/main/java/feast/core/validators/DataSourceValidator.java @@ -24,7 +24,11 @@ import feast.proto.core.DataSourceProto.DataSource; public class DataSourceValidator { - /** Validate if the given DataSource protobuf spec is valid. */ + /** + * Validate if the given DataSource protobuf spec is valid. + * + * @param spec spec to be validated + */ public static void validate(DataSource spec) { switch (spec.getType()) { case BATCH_FILE: diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java index 5f7ddd2..b6d7fcc 100644 --- a/core/src/main/java/feast/core/validators/Matchers.java +++ b/core/src/main/java/feast/core/validators/Matchers.java @@ -29,8 +29,8 @@ public class Matchers { private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); - private static Pattern VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD = - Pattern.compile("^[a-zA-Z0-9\\-_*]*$"); + private static Pattern VALID_CHARACTERS_REGEX_WITH_DASH = + Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_-]*$"); private static String ERROR_MESSAGE_TEMPLATE = "invalid value for %s resource, %s: %s"; @@ -70,15 +70,15 @@ public static void checkValidCharacters(String input, String resource) } } - public static void checkValidCharactersAllowAsterisk(String input, String resource) + public static void checkValidCharactersAllowDash(String input, String resource) throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD.matcher(input).matches()) { + if (!VALID_CHARACTERS_REGEX_WITH_DASH.matcher(input).matches()) { throw new IllegalArgumentException( String.format( ERROR_MESSAGE_TEMPLATE, resource, input, - "argument must only contain alphanumeric characters, dashes, underscores, or an asterisk.")); + "argument must only contain alphanumeric characters, dashes, or underscores.")); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java deleted file mode 100644 index 41faee7..0000000 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=http", - }) -public class CoreServiceAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/core/src/test/java/feast/core/logging/CoreLoggingIT.java deleted file mode 100644 index 0f137b4..0000000 --- a/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Streams; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.logging.entry.AuditLogEntryKind; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Status.Code; -import io.grpc.StatusRuntimeException; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest( - properties = { - "feast.logging.audit.enabled=true", - "feast.logging.audit.messageLogging.enabled=true", - "feast.logging.audit.messageLogging.destination=console" - }) -public class CoreLoggingIT extends BaseIT { - private static TestLogAppender testAuditLogAppender; - private static CoreServiceBlockingStub coreService; - private static CoreServiceFutureStub asyncCoreService; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int coreGrpcPort) - throws InterruptedException, ExecutionException { - LoggerContext logContext = (LoggerContext) LogManager.getContext(false); - // NOTE: As log appender state is shared across tests use a different method - // for each test and filter by method name to ensure that you only get logs - // for a specific test. - testAuditLogAppender = logContext.getConfiguration().getAppender("TestAuditLogAppender"); - - // Connect to core service. - Channel channel = - ManagedChannelBuilder.forAddress("localhost", coreGrpcPort).usePlaintext().build(); - coreService = CoreServiceGrpc.newBlockingStub(channel); - asyncCoreService = CoreServiceGrpc.newFutureStub(channel); - - // Preflight a request to core service stubs to verify connection - coreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()); - asyncCoreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()).get(); - } - - /** Check that messsage audit log are produced on service call */ - @Test - public void shouldProduceMessageAuditLogsOnCall() - throws InterruptedException, InvalidProtocolBufferException { - // Generate artifical load on feast core. - UpdateStoreRequest request = - UpdateStoreRequest.newBuilder().setStore(DataGenerator.getDefaultStore()).build(); - UpdateStoreResponse response = coreService.updateStore(request); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Check message audit logs are produced for each audit log. - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "UpdateStore"); - assertEquals(1, logJsonObjects.size()); - JsonObject logObj = logJsonObjects.get(0); - - // Extract & Check that request/response are returned correctly - String requestJson = logObj.getAsJsonObject("request").toString(); - UpdateStoreRequest.Builder gotRequest = UpdateStoreRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - UpdateStoreResponse.Builder gotResponse = UpdateStoreResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - } - - /** Check that message audit logs are produced when server encounters an error */ - @Test - public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { - // Send a bad request which should cause Core to error - ListFeatureTablesRequest request = - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) - .build(); - - boolean hasExpectedException = false; - Code statusCode = null; - try { - coreService.listFeatureTables(request); - } catch (StatusRuntimeException e) { - hasExpectedException = true; - statusCode = e.getStatus().getCode(); - } - assertTrue(hasExpectedException); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); - - assertEquals(1, logJsonObjects.size()); - JsonObject logJsonObject = logJsonObjects.get(0); - // Check correct status code is tracked on error. - assertEquals(logJsonObject.get("statusCode").getAsString(), statusCode.toString()); - } - - /** Check that expected message audit logs are produced when under load. */ - @Test - public void shouldProduceExpectedAuditLogsUnderLoad() - throws InterruptedException, ExecutionException { - // Generate artifical requests on core to simulate load. - int LOAD_SIZE = 40; // Total number of requests to send. - int BURST_SIZE = 5; // Number of requests to send at once. - - ListStoresRequest request = ListStoresRequest.getDefaultInstance(); - List responses = new LinkedList<>(); - for (int i = 0; i < LOAD_SIZE; i += 5) { - List> futures = new LinkedList<>(); - for (int j = 0; j < BURST_SIZE; j++) { - futures.add(asyncCoreService.listStores(request)); - } - - responses.addAll(Futures.allAsList(futures).get()); - } - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - - // Pull message audit logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListStores"); - assertEquals(responses.size(), logJsonObjects.size()); - - // Extract & Check that request/response are returned correctly - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - Streams.zip( - responses.stream(), - logJsonObjects.stream(), - (response, logObj) -> Pair.of(response, logObj)) - .forEach( - responseLogJsonPair -> { - ListStoresResponse response = responseLogJsonPair.getLeft(); - JsonObject logObj = responseLogJsonPair.getRight(); - - ListStoresRequest.Builder gotRequest = null; - ListStoresResponse.Builder gotResponse = null; - try { - String requestJson = logObj.getAsJsonObject("request").toString(); - gotRequest = ListStoresRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - gotResponse = ListStoresResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - }); - } - - /** - * Filter and Parse out Message Audit Logs from the given logsStrings for the given method name - */ - private List parseMessageJsonLogObjects(List logsStrings, String methodName) { - JsonParser jsonParser = new JsonParser(); - // copy to prevent concurrent modification. - return logsStrings.stream() - .map(logJSON -> jsonParser.parse(logJSON).getAsJsonObject()) - // Filter to only include message audit logs - .filter( - logObj -> - logObj - .getAsJsonPrimitive("kind") - .getAsString() - .equals(AuditLogEntryKind.MESSAGE.toString()) - // filter by method name to ensure logs from other tests do not interfere with - // test - && logObj.get("method").getAsString().equals(methodName)) - .collect(Collectors.toList()); - } -} diff --git a/core/src/test/java/feast/core/service/ProjectServiceTest.java b/core/src/test/java/feast/core/service/ProjectServiceTest.java index e09580f..afff1e5 100644 --- a/core/src/test/java/feast/core/service/ProjectServiceTest.java +++ b/core/src/test/java/feast/core/service/ProjectServiceTest.java @@ -16,10 +16,8 @@ */ package feast.core.service; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; import feast.core.dao.ProjectRepository; @@ -29,16 +27,12 @@ import java.util.Optional; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.mockito.Mock; public class ProjectServiceTest { @Mock private ProjectRepository projectRepository; - @Rule public final ExpectedException expectedException = ExpectedException.none(); - private ProjectService projectService; @Before @@ -75,8 +69,9 @@ public void shouldArchiveProjectIfItExists() { @Test public void shouldNotArchiveDefaultProject() { - expectedException.expect(IllegalArgumentException.class); - this.projectService.archiveProject(Project.DEFAULT_NAME); + assertThrows( + IllegalArgumentException.class, + () -> this.projectService.archiveProject(Project.DEFAULT_NAME)); } @Test(expected = IllegalArgumentException.class) diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 8851d87..d6d0f74 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -170,7 +170,7 @@ public void shouldThrowExceptionGivenWildcardProject() { equalTo( String.format( "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", + + "argument must only contain alphanumeric characters, dashes, or underscores.", filter.getProject()))); } } @@ -226,7 +226,7 @@ public void shouldThrowExceptionGivenWildcardProject() { equalTo( String.format( "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", + + "argument must only contain alphanumeric characters, dashes, or underscores.", filter.getProject()))); } } diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java index c44bf50..ba965a7 100644 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ b/core/src/test/java/feast/core/util/TypeConversionTest.java @@ -17,8 +17,8 @@ package feast.core.util; import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.*; import com.google.protobuf.Timestamp; import java.util.*; diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java index 1733212..559330a 100644 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ b/core/src/test/java/feast/core/validators/MatchersTest.java @@ -19,14 +19,12 @@ import static feast.core.validators.Matchers.checkLowerSnakeCase; import static feast.core.validators.Matchers.checkUpperSnakeCase; import static feast.core.validators.Matchers.checkValidClassPath; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.base.Strings; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; public class MatchersTest { - @Rule public final ExpectedException exception = ExpectedException.none(); @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { @@ -42,15 +40,15 @@ public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { @Test public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( + String in = "redis"; + assertThrows( + IllegalArgumentException.class, + () -> checkUpperSnakeCase(in, "featuretable"), Strings.lenientFormat( "invalid value for %s resource, %s: %s", "featuretable", "redis", "argument must be in upper snake case, and cannot include any special characters.")); - String in = "redis"; - checkUpperSnakeCase(in, "featuretable"); } @Test @@ -61,15 +59,15 @@ public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() { @Test public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( + String in = "Invalid_feature name"; + assertThrows( + IllegalArgumentException.class, + () -> checkLowerSnakeCase(in, "feature"), Strings.lenientFormat( "invalid value for %s resource, %s: %s", "feature", "Invalid_feature name", "argument must be in lower snake case, and cannot include any special characters.")); - String in = "Invalid_feature name"; - checkLowerSnakeCase(in, "feature"); } @Test @@ -80,13 +78,11 @@ public void checkValidClassPathSuccess() { @Test public void checkValidClassPathEmpty() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("", "FeatureTable"); + assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("", "FeatureTable")); } @Test public void checkValidClassPathDigits() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("123", "FeatureTable"); + assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("123", "FeatureTable")); } } diff --git a/core/src/test/resources/keto/docker-compose.yml b/core/src/test/resources/keto/docker-compose.yml index 4d0fc43..714f83e 100644 --- a/core/src/test/resources/keto/docker-compose.yml +++ b/core/src/test/resources/keto/docker-compose.yml @@ -7,6 +7,7 @@ services: image: oryd/keto:v0.4.3-alpha.2 environment: - DSN=postgres://keto:keto@db:5432/keto?sslmode=disable + - SERVE_CORS_ENABLED=true command: - serve ports: @@ -32,11 +33,11 @@ services: adaptor: depends_on: - - keto + - keto image: gcr.io/kf-feast/feast-keto-auth-server:latest environment: SERVER_PORT: 8080 KETO_URL: http://keto:4466 ports: - - 8080 - restart: on-failure \ No newline at end of file + - 8080 + restart: on-failure diff --git a/datatypes/java/README.md b/datatypes/java/README.md index 30b568e..7fc355f 100644 --- a/datatypes/java/README.md +++ b/datatypes/java/README.md @@ -16,7 +16,7 @@ Dependency Coordinates dev.feast datatypes-java - 0.10.0-SNAPSHOT + 0.26.2 ``` diff --git a/deps/feast b/deps/feast index 8010d2f..2541c91 160000 --- a/deps/feast +++ b/deps/feast @@ -1 +1 @@ -Subproject commit 8010d2f35d3f876db54a31b8012b13009cd5eba2 +Subproject commit 2541c91c9238ef09ffd74e45e74116a31d7f2daa diff --git a/docs/architecture.png b/docs/architecture.png new file mode 100644 index 0000000..3bc348b Binary files /dev/null and b/docs/architecture.png differ diff --git a/infra/architecture.png b/infra/architecture.png new file mode 100644 index 0000000..2667fd1 Binary files /dev/null and b/infra/architecture.png differ diff --git a/infra/charts/feast-core/Chart.yaml b/infra/charts/feast-core/Chart.yaml new file mode 100644 index 0000000..ac52a32 --- /dev/null +++ b/infra/charts/feast-core/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +description: "Feast Core: Feature registry for Feast." +name: feast-core +version: 0.26.2 +appVersion: 0.26.2 +keywords: +- machine learning +- big data +- mlops +home: https://github.com/feast-dev/feast-java diff --git a/infra/charts/feast-core/README.md b/infra/charts/feast-core/README.md new file mode 100644 index 0000000..bae6786 --- /dev/null +++ b/infra/charts/feast-core/README.md @@ -0,0 +1,68 @@ +feast-core +========== +Feast Core: Feature registry for Feast. + +Current chart version is `0.26.2` + +Source code can be found [here](https://github.com/feast-dev/feast-java) + + + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. | +| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | +| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | +| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | +| envOverrides | object | `{}` | Extra environment variables to set | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"gcr.io/kf-feast/feast-core"` | Docker image repository | +| image.tag | string | `"develop"` | Image tag | +| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | +| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges | +| ingress.http.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service | +| ingress.http.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.http.class | string | `"nginx"` | Which ingress controller to use | +| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.http.whitelist | string | `""` | Allowed client IP source ranges | +| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` | +| livenessProbe.enabled | bool | `false` | Flag to enabled the probe | +| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated | +| livenessProbe.periodSeconds | int | `10` | How often to perform the probe | +| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| livenessProbe.timeoutSeconds | int | `5` | When the probe times out | +| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | +| logType | string | `"Console"` | Log format, either `JSON` or `Console` | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| podLabels | object | `{}` | Labels to be added to Feast Core pods | +| postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres | +| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | +| readinessProbe.enabled | bool | `true` | Flag to enabled the probe | +| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated | +| readinessProbe.periodSeconds | int | `10` | How often to perform the probe | +| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| readinessProbe.timeoutSeconds | int | `10` | When the probe times out | +| replicaCount | int | `1` | Number of pods that will be created | +| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) | +| secrets | list | `[]` | List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. | +| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.grpc.port | int | `6565` | Service port for GRPC requests | +| service.grpc.targetPort | int | `6565` | Container port serving GRPC requests | +| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.http.port | int | `80` | Service port for HTTP requests | +| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics | +| service.type | string | `"ClusterIP"` | Kubernetes service type | diff --git a/infra/charts/feast-core/templates/_helpers.tpl b/infra/charts/feast-core/templates/_helpers.tpl new file mode 100644 index 0000000..7c095e1 --- /dev/null +++ b/infra/charts/feast-core/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "feast-core.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "feast-core.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "feast-core.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "feast-core.labels" -}} +app.kubernetes.io/name: {{ include "feast-core.name" . }} +helm.sh/chart: {{ include "feast-core.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/infra/charts/feast-core/templates/_ingress.yaml b/infra/charts/feast-core/templates/_ingress.yaml new file mode 100644 index 0000000..5bed6df --- /dev/null +++ b/infra/charts/feast-core/templates/_ingress.yaml @@ -0,0 +1,68 @@ +{{- /* +This takes an array of three values: +- the top context +- the feast component +- the service protocol +- the ingress context +*/ -}} +{{- define "feast.ingress" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +apiVersion: extensions/v1beta1 +kind: Ingress +{{ include "feast.ingress.metadata" . }} +spec: + rules: + {{- range $host := $ingressValues.hosts }} + - host: {{ $host }} + http: + paths: + - path: / + backend: + serviceName: {{ include (printf "feast-%s.fullname" $component) $top }} + servicePort: {{ index $top.Values "service" $protocol "port" }} + {{- end }} +{{- if $ingressValues.https.enabled }} + tls: + {{- range $host := $ingressValues.hosts }} + - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }} + hosts: + - {{ $host }} + {{- end }} +{{- end -}} +{{- end -}} + +{{- define "feast.ingress.metadata" -}} +{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }} +{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}} +{{- toYaml (merge $overrides $commonMetadata) -}} +{{- end -}} + +{{- define "feast.ingress.metadata-overrides" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +{{- $commonFullname := include "common.fullname" $top }} +metadata: + name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }} + annotations: + kubernetes.io/ingress.class: {{ $ingressValues.class | quote }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }} + nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }} + nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user" + nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri" + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}} + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }} + # TODO: Allow choice of GRPC/GRPCS + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + {{- end }} + {{- if $ingressValues.annotations -}} + {{ include "common.annote" $ingressValues.annotations | indent 4 }} + {{- end }} +{{- end -}} diff --git a/infra/charts/feast-core/templates/configmap.yaml b/infra/charts/feast-core/templates/configmap.yaml new file mode 100644 index 0000000..7e0185a --- /dev/null +++ b/infra/charts/feast-core/templates/configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + application-generated.yaml: | +{{- if index .Values "application-generated.yaml" "enabled" }} + spring: + datasource: + url: jdbc:postgresql://{{ .Release.Name }}-postgresql:5432/postgres + + server: + port: {{ .Values.service.http.targetPort }} +{{- end }} + + application-override.yaml: | +{{- if index .Values "application-override.yaml" "enabled" }} +{{- toYaml (index .Values "application-override.yaml") | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/infra/charts/feast-core/templates/deployment.yaml b/infra/charts/feast-core/templates/deployment.yaml new file mode 100644 index 0000000..a24672d --- /dev/null +++ b/infra/charts/feast-core/templates/deployment.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "feast-core.name" . }} + component: core + release: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.prometheus.enabled }} + prometheus.io/path: /metrics + prometheus.io/port: "{{ .Values.service.http.targetPort }}" + prometheus.io/scrape: "true" + {{- end }} + labels: + app: {{ template "feast-core.name" . }} + component: core + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: {{ template "feast-core.fullname" . }}-config + configMap: + name: {{ template "feast-core.fullname" . }} + - name: {{ template "feast-core.fullname" . }}-secret + secret: + secretName: {{ template "feast-core.fullname" . }} + {{- range $secret := .Values.secrets }} + - name: {{ $secret }} + secret: + secretName: {{ $secret }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + + volumeMounts: + - name: {{ template "feast-core.fullname" . }}-config + mountPath: /etc/feast + - name: {{ template "feast-core.fullname" . }}-secret + mountPath: /etc/secrets/feast + readOnly: true + {{- range $secret := .Values.secrets }} + - name: {{ $secret }} + mountPath: "/etc/secrets/{{ $secret }}" + readOnly: true + {{- end }} + + env: + - name: LOG_TYPE + value: {{ .Values.logType | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + + {{- if .Values.postgresql.existingSecret }} + - name: SPRING_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.existingSecret }} + key: postgresql-password + {{- end }} + + {{- if .Values.javaOpts }} + - name: JAVA_TOOL_OPTIONS + value: {{ .Values.javaOpts }} + {{- end }} + + {{- range $key, $value := .Values.envOverrides }} + - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} + {{- if eq (kindOf $value) "map" }} + valueFrom: + {{- toYaml $value | nindent 12}} + {{- else }} + value: {{ $value | quote }} + {{- end}} + {{- end }} + + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location= + {{- if index .Values "application.yaml" "enabled" -}} + classpath:/application.yml + {{- end }} + {{- if index .Values "application-generated.yaml" "enabled" -}} + ,file:/etc/feast/application-generated.yaml + {{- end }} + {{- if index .Values "application-secret.yaml" "enabled" -}} + ,file:/etc/secrets/feast/application-secret.yaml + {{- end }} + {{- if index .Values "application-override.yaml" "enabled" -}} + ,file:/etc/feast/application-override.yaml + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.http.targetPort }} + - name: grpc + containerPort: {{ .Values.service.grpc.targetPort }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 10 }} diff --git a/infra/charts/feast-core/templates/ingress.yaml b/infra/charts/feast-core/templates/ingress.yaml new file mode 100644 index 0000000..7f453e1 --- /dev/null +++ b/infra/charts/feast-core/templates/ingress.yaml @@ -0,0 +1,7 @@ +{{- if .Values.ingress.http.enabled -}} +{{ template "feast.ingress" (list . "core" "http" .Values.ingress.http) }} +{{- end }} +--- +{{ if .Values.ingress.grpc.enabled -}} +{{ template "feast.ingress" (list . "core" "grpc" .Values.ingress.grpc) }} +{{- end }} diff --git a/infra/charts/feast-core/templates/secret.yaml b/infra/charts/feast-core/templates/secret.yaml new file mode 100644 index 0000000..dd33e2d --- /dev/null +++ b/infra/charts/feast-core/templates/secret.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +stringData: + application-secret.yaml: | +{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }} diff --git a/infra/charts/feast-core/templates/service.yaml b/infra/charts/feast-core/templates/service.yaml new file mode 100644 index 0000000..1561398 --- /dev/null +++ b/infra/charts/feast-core/templates/service.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.service.annotations }} + annotations: + {{ toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.http.port }} + targetPort: {{ .Values.service.http.targetPort }} + {{- if .Values.service.http.nodePort }} + nodePort: {{ .Values.service.http.nodePort }} + {{- end }} + - name: grpc + port: {{ .Values.service.grpc.port }} + targetPort: {{ .Values.service.grpc.targetPort }} + {{- if .Values.service.grpc.nodePort }} + nodePort: {{ .Values.service.grpc.nodePort }} + {{- end }} + selector: + app: {{ template "feast-core.name" . }} + component: core + release: {{ .Release.Name }} + diff --git a/infra/charts/feast-core/values.yaml b/infra/charts/feast-core/values.yaml new file mode 100644 index 0000000..423373e --- /dev/null +++ b/infra/charts/feast-core/values.yaml @@ -0,0 +1,147 @@ +# replicaCount -- Number of pods that will be created +replicaCount: 1 + +image: + # image.repository -- Docker image repository + repository: gcr.io/kf-feast/feast-core + # image.tag -- Image tag + tag: develop + # image.pullPolicy -- Image pull policy + pullPolicy: IfNotPresent + +application.yaml: + # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. + enabled: true + +application-generated.yaml: + # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. + enabled: true + +# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. +application-secret.yaml: + enabled: true + +# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` +application-override.yaml: + enabled: true + +postgresql: + # postgresql.existingSecret -- Existing secret to use for authenticating to Postgres + existingSecret: "" + +# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` +javaOpts: + +# logType -- Log format, either `JSON` or `Console` +logType: Console +# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` +logLevel: WARN + +prometheus: + # prometheus.enabled -- Flag to enable scraping of Feast Core metrics + enabled: true + +# By default we disable the liveness probe, since if the DB fails restarting core will not result +# in application healing. +livenessProbe: + # livenessProbe.enabled -- Flag to enabled the probe + enabled: false + # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 60 + # livenessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # livenessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 5 + # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +readinessProbe: + # readinessProbe.enabled -- Flag to enabled the probe + enabled: true + # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 20 + # readinessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # readinessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 10 + # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +service: + # service.type -- Kubernetes service type + type: ClusterIP + http: + # service.http.port -- Service port for HTTP requests + port: 80 + # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics + targetPort: 8080 + # service.http.nodePort -- Port number that each cluster node will listen to + nodePort: + grpc: + # service.grpc.port -- Service port for GRPC requests + port: 6565 + # service.grpc.targetPort -- Container port serving GRPC requests + targetPort: 6565 + # service.grpc.nodePort -- Port number that each cluster node will listen to + nodePort: + +ingress: + grpc: + # ingress.grpc.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.grpc.class -- Which ingress controller to use + class: nginx + # ingress.grpc.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.grpc.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.grpc.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.grpc.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.grpc.auth.enabled -- Flag to enable auth + enabled: false + http: + # ingress.http.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.http.class -- Which ingress controller to use + class: nginx + # ingress.http.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.http.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.http.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.http.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.http.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.http.auth.enabled -- Flag to enable auth + enabled: false + # ingress.http.auth.authUrl -- URL to an existing authentication service + authUrl: http://auth-server.auth-ns.svc.cluster.local/auth + +# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) +resources: {} + +# nodeSelector -- Node labels for pod assignment +nodeSelector: {} + +# envOverrides -- Extra environment variables to set +envOverrides: {} + +# secrets -- List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. +secrets: [] + +# podLabels -- Labels to be added to Feast Core pods +podLabels: {} diff --git a/infra/charts/feast-serving/Chart.yaml b/infra/charts/feast-serving/Chart.yaml new file mode 100644 index 0000000..e9a4c41 --- /dev/null +++ b/infra/charts/feast-serving/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +description: "Feast Serving: Online feature serving service for Feast" +name: feast-serving +version: 0.26.2 +appVersion: 0.26.2 +keywords: +- machine learning +- big data +- mlops +home: https://github.com/feast-dev/feast-java diff --git a/infra/charts/feast-serving/README.md b/infra/charts/feast-serving/README.md new file mode 100644 index 0000000..66a982c --- /dev/null +++ b/infra/charts/feast-serving/README.md @@ -0,0 +1,71 @@ +feast-serving +============= +Feast Serving: Online feature serving service for Feast + +Current chart version is `0.26.2` + +Source code can be found [here](https://github.com/feast-dev/feast-java) + + + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration. | +| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | +| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | +| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | +| envOverrides | object | `{}` | Extra environment variables to set | +| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow | +| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key Cloud service account JSON key file. | +| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | +| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"gcr.io/kf-feast/feast-serving"` | Docker image repository | +| image.tag | string | `"develop"` | Image tag | +| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | +| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges | +| ingress.http.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service | +| ingress.http.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.http.class | string | `"nginx"` | Which ingress controller to use | +| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.http.whitelist | string | `""` | Allowed client IP source ranges | +| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` | +| livenessProbe.enabled | bool | `true` | Flag to enabled the probe | +| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated | +| livenessProbe.periodSeconds | int | `10` | How often to perform the probe | +| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| livenessProbe.timeoutSeconds | int | `5` | When the probe times out | +| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | +| logType | string | `"Console"` | Log format, either `JSON` or `Console` | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| podLabels | object | `{}` | Labels to be added to Feast Serving pods | +| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | +| readinessProbe.enabled | bool | `true` | Flag to enabled the probe | +| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| readinessProbe.initialDelaySeconds | int | `15` | Delay before the probe is initiated | +| readinessProbe.periodSeconds | int | `10` | How often to perform the probe | +| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| readinessProbe.timeoutSeconds | int | `10` | When the probe times out | +| replicaCount | int | `1` | Number of pods that will be created | +| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) | +| secrets | list | `[]` | List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. | +| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.grpc.port | int | `6566` | Service port for GRPC requests | +| service.grpc.targetPort | int | `6566` | Container port serving GRPC requests | +| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.http.port | int | `80` | Service port for HTTP requests | +| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics | +| service.type | string | `"ClusterIP"` | Kubernetes service type | diff --git a/infra/charts/feast-serving/templates/_helpers.tpl b/infra/charts/feast-serving/templates/_helpers.tpl new file mode 100644 index 0000000..49abb6b --- /dev/null +++ b/infra/charts/feast-serving/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "feast-serving.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "feast-serving.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "feast-serving.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "feast-serving.labels" -}} +app.kubernetes.io/name: {{ include "feast-serving.name" . }} +helm.sh/chart: {{ include "feast-serving.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/infra/charts/feast-serving/templates/_ingress.yaml b/infra/charts/feast-serving/templates/_ingress.yaml new file mode 100644 index 0000000..5bed6df --- /dev/null +++ b/infra/charts/feast-serving/templates/_ingress.yaml @@ -0,0 +1,68 @@ +{{- /* +This takes an array of three values: +- the top context +- the feast component +- the service protocol +- the ingress context +*/ -}} +{{- define "feast.ingress" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +apiVersion: extensions/v1beta1 +kind: Ingress +{{ include "feast.ingress.metadata" . }} +spec: + rules: + {{- range $host := $ingressValues.hosts }} + - host: {{ $host }} + http: + paths: + - path: / + backend: + serviceName: {{ include (printf "feast-%s.fullname" $component) $top }} + servicePort: {{ index $top.Values "service" $protocol "port" }} + {{- end }} +{{- if $ingressValues.https.enabled }} + tls: + {{- range $host := $ingressValues.hosts }} + - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }} + hosts: + - {{ $host }} + {{- end }} +{{- end -}} +{{- end -}} + +{{- define "feast.ingress.metadata" -}} +{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }} +{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}} +{{- toYaml (merge $overrides $commonMetadata) -}} +{{- end -}} + +{{- define "feast.ingress.metadata-overrides" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +{{- $commonFullname := include "common.fullname" $top }} +metadata: + name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }} + annotations: + kubernetes.io/ingress.class: {{ $ingressValues.class | quote }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }} + nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }} + nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user" + nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri" + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}} + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }} + # TODO: Allow choice of GRPC/GRPCS + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + {{- end }} + {{- if $ingressValues.annotations -}} + {{ include "common.annote" $ingressValues.annotations | indent 4 }} + {{- end }} +{{- end -}} diff --git a/infra/charts/feast-serving/templates/configmap.yaml b/infra/charts/feast-serving/templates/configmap.yaml new file mode 100644 index 0000000..011b4cb --- /dev/null +++ b/infra/charts/feast-serving/templates/configmap.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + application-generated.yaml: | +{{- if index .Values "application-generated.yaml" "enabled" }} + feast: + core-host: {{ .Release.Name }}-feast-core + + stores: + - name: online + type: REDIS + config: + host: {{ .Release.Name }}-redis-master + port: 6379 + subscriptions: + - name: "*" + project: "*" + version: "*" + + job_store: + redis_host: {{ .Release.Name }}-redis-master + redis_port: 6379 + + server: + port: {{ .Values.service.http.targetPort }} +{{- end }} + + application-override.yaml: | +{{- if index .Values "application-override.yaml" "enabled" }} +{{- toYaml (index .Values "application-override.yaml") | nindent 4 }} +{{- end }} diff --git a/infra/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast-serving/templates/deployment.yaml new file mode 100644 index 0000000..1d6e3aa --- /dev/null +++ b/infra/charts/feast-serving/templates/deployment.yaml @@ -0,0 +1,148 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "feast-serving.name" . }} + component: serving + release: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.prometheus.enabled }} + prometheus.io/path: /metrics + prometheus.io/port: "{{ .Values.service.http.targetPort }}" + prometheus.io/scrape: "true" + {{- end }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: {{ template "feast-serving.fullname" . }}-config + configMap: + name: {{ template "feast-serving.fullname" . }} + - name: {{ template "feast-serving.fullname" . }}-secret + secret: + secretName: {{ template "feast-serving.fullname" . }} + {{- range $secret := .Values.secrets }} + - name: {{ $secret }} + secret: + secretName: {{ $secret }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + + volumeMounts: + - name: {{ template "feast-serving.fullname" . }}-config + mountPath: /etc/feast + - name: {{ template "feast-serving.fullname" . }}-secret + mountPath: /etc/secrets/feast + readOnly: true + {{- range $secret := .Values.secrets }} + - name: {{ $secret }} + mountPath: "/etc/secrets/{{ $secret }}" + readOnly: true + {{- end }} + + env: + - name: LOG_TYPE + value: {{ .Values.logType | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + + {{- if .Values.javaOpts }} + - name: JAVA_TOOL_OPTIONS + value: {{ .Values.javaOpts }} + {{- end }} + + {{- range $key, $value := .Values.envOverrides }} + - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} + {{- if eq (kindOf $value) "map" }} + valueFrom: + {{- toYaml $value | nindent 12 }} + {{- else }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + + command: + - java + - -jar + - /opt/feast/feast-serving.jar + - --spring.config.location= + {{- if index .Values "application.yaml" "enabled" -}} + classpath:/application.yml + {{- end }} + {{- if index .Values "application-generated.yaml" "enabled" -}} + ,file:/etc/feast/application-generated.yaml + {{- end }} + {{- if index .Values "application-secret.yaml" "enabled" -}} + ,file:/etc/secrets/feast/application-secret.yaml + {{- end }} + {{- if index .Values "application-override.yaml" "enabled" -}} + ,file:/etc/feast/application-override.yaml + {{- end }} + + ports: + - name: http + containerPort: {{ .Values.service.http.targetPort }} + - name: grpc + containerPort: {{ .Values.service.grpc.targetPort }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - "grpc-health-probe" + - "-addr=:{{ .Values.service.grpc.targetPort }}" + - "-connect-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" + - "-rpc-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - "grpc-health-probe" + - "-addr=:{{ .Values.service.grpc.targetPort }}" + - "-connect-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" + - "-rpc-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 10 }} diff --git a/infra/charts/feast-serving/templates/ingress.yaml b/infra/charts/feast-serving/templates/ingress.yaml new file mode 100644 index 0000000..1bcd176 --- /dev/null +++ b/infra/charts/feast-serving/templates/ingress.yaml @@ -0,0 +1,7 @@ +{{- if .Values.ingress.http.enabled -}} +{{ template "feast.ingress" (list . "serving" "http" .Values.ingress.http) }} +{{- end }} +--- +{{ if .Values.ingress.grpc.enabled -}} +{{ template "feast.ingress" (list . "serving" "grpc" .Values.ingress.grpc) }} +{{- end }} diff --git a/infra/charts/feast-serving/templates/secret.yaml b/infra/charts/feast-serving/templates/secret.yaml new file mode 100644 index 0000000..2ccbccf --- /dev/null +++ b/infra/charts/feast-serving/templates/secret.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +stringData: + application-secret.yaml: | +{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }} diff --git a/infra/charts/feast-serving/templates/service.yaml b/infra/charts/feast-serving/templates/service.yaml new file mode 100644 index 0000000..7afdd31 --- /dev/null +++ b/infra/charts/feast-serving/templates/service.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 2 }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.http.port }} + targetPort: {{ .Values.service.http.targetPort }} + {{- if .Values.service.http.nodePort }} + nodePort: {{ .Values.service.http.nodePort }} + {{- end }} + - name: grpc + port: {{ .Values.service.grpc.port }} + targetPort: {{ .Values.service.grpc.targetPort }} + {{- if .Values.service.grpc.nodePort }} + nodePort: {{ .Values.service.grpc.nodePort }} + {{- end }} + selector: + app: {{ template "feast-serving.name" . }} + component: serving + release: {{ .Release.Name }} diff --git a/infra/charts/feast-serving/values.yaml b/infra/charts/feast-serving/values.yaml new file mode 100644 index 0000000..06dbb85 --- /dev/null +++ b/infra/charts/feast-serving/values.yaml @@ -0,0 +1,154 @@ +# replicaCount -- Number of pods that will be created +replicaCount: 1 + +image: + # image.repository -- Docker image repository + repository: gcr.io/kf-feast/feast-serving + # image.tag -- Image tag + tag: develop + # image.pullPolicy -- Image pull policy + pullPolicy: IfNotPresent + +application.yaml: + # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. + enabled: true + +application-generated.yaml: + # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration. + enabled: true + +# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. +application-secret.yaml: + enabled: true + +# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` +application-override.yaml: + enabled: true + +gcpServiceAccount: + # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key + # Cloud service account JSON key file. + enabled: false + existingSecret: + # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account + name: feast-gcp-service-account + # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account) + key: credentials.json + +# gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow +gcpProjectId: "" + +# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` +javaOpts: + +# logType -- Log format, either `JSON` or `Console` +logType: Console +# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` +logLevel: WARN + +prometheus: + # prometheus.enabled -- Flag to enable scraping of Feast Core metrics + enabled: true + +livenessProbe: + # livenessProbe.enabled -- Flag to enabled the probe + enabled: true + # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 60 + # livenessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # livenessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 5 + # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +readinessProbe: + # readinessProbe.enabled -- Flag to enabled the probe + enabled: true + # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 15 + # readinessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # readinessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 10 + # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +service: + # service.type -- Kubernetes service type + type: ClusterIP + http: + # service.http.port -- Service port for HTTP requests + port: 80 + # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics + targetPort: 8080 + # service.http.nodePort -- Port number that each cluster node will listen to + nodePort: + grpc: + # service.grpc.port -- Service port for GRPC requests + port: 6566 + # service.grpc.targetPort -- Container port serving GRPC requests + targetPort: 6566 + # service.grpc.nodePort -- Port number that each cluster node will listen to + nodePort: + +ingress: + grpc: + # ingress.grpc.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.grpc.class -- Which ingress controller to use + class: nginx + # ingress.grpc.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.grpc.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.grpc.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.grpc.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.grpc.auth.enabled -- Flag to enable auth + enabled: false + http: + # ingress.http.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.http.class -- Which ingress controller to use + class: nginx + # ingress.http.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.http.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.http.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.http.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.http.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.http.auth.enabled -- Flag to enable auth + enabled: false + # ingress.http.auth.authUrl -- URL to an existing authentication service + authUrl: http://auth-server.auth-ns.svc.cluster.local/auth + +# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) +resources: {} + +# nodeSelector -- Node labels for pod assignment +nodeSelector: {} + +# envOverrides -- Extra environment variables to set +envOverrides: {} + +# secrets -- List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. +secrets: [] + +# podLabels -- Labels to be added to Feast Serving pods +podLabels: {} diff --git a/infra/scripts/install-helm.sh b/infra/scripts/install-helm.sh new file mode 100755 index 0000000..3686f9d --- /dev/null +++ b/infra/scripts/install-helm.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm +readonly HELM_TARBALL="helm-${HELM_VERSION}-linux-amd64.tar.gz" +readonly STABLE_REPO_URL=https://charts.helm.sh/stable +readonly INCUBATOR_REPO_URL=https://charts.helm.sh/incubator +curl -s "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | tar -C /tmp -xz +sudo mv /tmp/linux-amd64/helm /usr/bin/helm +helm init --client-only +helm repo add incubator "$INCUBATOR_REPO_URL" \ No newline at end of file diff --git a/infra/scripts/publish-java-sdk.sh b/infra/scripts/publish-java-sdk.sh index 91123c8..bd06410 100755 --- a/infra/scripts/publish-java-sdk.sh +++ b/infra/scripts/publish-java-sdk.sh @@ -69,4 +69,4 @@ gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key echo "============================================================" echo "Deploying Java SDK with revision: $REVISION" echo "============================================================" -mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy +mvn --projects .,datatypes/java,common,sdk/java -Drevision=$REVISION --batch-mode clean deploy \ No newline at end of file diff --git a/infra/scripts/push-helm-charts.sh b/infra/scripts/push-helm-charts.sh new file mode 100755 index 0000000..4f45a9a --- /dev/null +++ b/infra/scripts/push-helm-charts.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +if [ $# -ne 1 ]; then + echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0" + exit 1 +fi + +bucket=gs://feast-helm-charts/ +repo_url=https://feast-helm-charts.storage.googleapis.com/ + +helm plugin install https://github.com/hayorov/helm-gcs.git --version 0.2.2 || true + +helm repo add feast-helm-chart-repo $bucket + +helm package infra/charts/feast-core +helm package infra/charts/feast-serving + +helm gcs push --public --force feast-core-${1}.tgz feast-helm-chart-repo +helm gcs push --public --force feast-serving-${1}.tgz feast-helm-chart-repo \ No newline at end of file diff --git a/infra/scripts/validate-helm-chart-versions.sh b/infra/scripts/validate-helm-chart-versions.sh new file mode 100755 index 0000000..8269665 --- /dev/null +++ b/infra/scripts/validate-helm-chart-versions.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +function finish { + echo "Please ensure the Chart.yaml have the version ${1}" + exit +} + +trap "finish $1" ERR + +set -e + +if [ $# -ne 1 ]; then + echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0" + exit 1 +fi + +# Get project root +PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) + +echo "Trying to find version ${1} in the feast-core Chart.yaml. Exiting if not found." +grep "version: ${1}" "${PROJECT_ROOT_DIR}/infra/charts/feast-core/Chart.yaml" + +echo "Trying to find version ${1} in the feast-serving Chart.yaml. Exiting if not found." +grep "version: ${1}" "${PROJECT_ROOT_DIR}/infra/charts/feast-serving/Chart.yaml" + +echo "Success! All versions found!" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 4feb250..50d9b95 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ - 0.10.0-SNAPSHOT + 0.26.2 https://github.com/feast-dev/feast UTF-8 @@ -269,7 +269,38 @@ ${grpc.version} test - + + + + org.apache.arrow + arrow-java-root + 5.0.0 + pom + + + + + org.apache.arrow + arrow-vector + 5.0.0 + + + + + org.apache.arrow + arrow-memory + 5.0.0 + pom + + + + + org.apache.arrow + arrow-memory-netty + 5.0.0 + runtime + + io.swagger @@ -543,6 +574,9 @@ + + feast.proto.*:io.grpc.*:org.tensorflow.* + com.diffplug.spotless @@ -566,10 +600,6 @@ ${license.content} - - 2.7.2 - ${parent.basedir}/.scalafmt.conf - @@ -695,7 +725,7 @@ - ${groupId}:${artifactId} + ${project.groupId}:${project.artifactId} ${project.build.outputDirectory} diff --git a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java b/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java index bd3b34b..94c779c 100644 --- a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java +++ b/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java @@ -26,15 +26,23 @@ public abstract class SecurityConfig { /** * Enables authentication If specified, the call credentials used to provide credentials to * authenticate with Feast. + * + * @return credentials */ public abstract Optional getCredentials(); - /** Whether to use TLS transport security is use when connecting to Feast. */ + /** + * Whether to use TLS transport security is use when connecting to Feast. + * + * @return true if enabled + */ public abstract boolean isTLSEnabled(); /** * If specified and TLS is enabled, provides path to TLS certificate use the verify Service * identity. + * + * @return certificate path */ public abstract Optional getCertificatePath(); diff --git a/serving/README.md b/serving/README.md index ab530bb..cce8c7d 100644 --- a/serving/README.md +++ b/serving/README.md @@ -88,3 +88,14 @@ with open("/tmp/000000000000.avro", "rb") as f: print(df.head(5)) EOF ``` +#### Working with Feast 0.10+ +Feast serving supports reading feature values materialized into Redis by feast 0.10+. To configure this, feast-serving +needs to be able to read the registry file for the project. +The location of the registry file can be specified in the `application.yml` like so: +```yaml +feast: + registry: "src/test/resources/docker-compose/feast10/registry.db" +``` + +This changes the behaviour of feast-serving to look up feature view definitions and specifications from the registry file instead +of the core service. \ No newline at end of file diff --git a/serving/pom.xml b/serving/pom.xml index b8f675d..981c23c 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -84,12 +84,32 @@ ${project.version} + + dev.feast + feast-storage-connector-bigtable + ${project.version} + + + + dev.feast + feast-storage-connector-cassandra + ${project.version} + + dev.feast feast-common ${project.version} - + + + com.google.cloud + google-cloud-bigtable-emulator + 0.130.2 + test + + + org.slf4j @@ -129,6 +149,14 @@ spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-test + 2.3.1.RELEASE + test + + io.grpc @@ -234,6 +262,44 @@ test + + + org.apache.avro + avro + 1.10.2 + + + + + org.apache.arrow + arrow-java-root + 5.0.0 + pom + + + + + org.apache.arrow + arrow-vector + 5.0.0 + + + + + org.apache.arrow + arrow-memory + 5.0.0 + pom + + + + + org.apache.arrow + arrow-memory-netty + 5.0.0 + runtime + + com.fasterxml.jackson.dataformat @@ -307,6 +373,12 @@ 1.15.1 test + + org.testcontainers + gcloud + 1.15.2 + test + org.awaitility awaitility diff --git a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java index 2bc9743..cdf791c 100644 --- a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java +++ b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java @@ -18,11 +18,13 @@ import java.util.concurrent.ScheduledExecutorService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; import org.springframework.stereotype.Component; @Component +@ConditionalOnBean(CoreCondition.class) public class ContextClosedHandler implements ApplicationListener { @Autowired ScheduledExecutorService executor; diff --git a/serving/src/main/java/feast/serving/config/CoreCondition.java b/serving/src/main/java/feast/serving/config/CoreCondition.java new file mode 100644 index 0000000..10dabfa --- /dev/null +++ b/serving/src/main/java/feast/serving/config/CoreCondition.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from Core service. + */ +public class CoreCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") == null; + } +} diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index bf04845..82db07c 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -21,24 +21,20 @@ // https://www.baeldung.com/configuration-properties-in-spring-boot // https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; import feast.common.auth.config.SecurityProperties; import feast.common.auth.config.SecurityProperties.AuthenticationProperties; import feast.common.auth.config.SecurityProperties.AuthorizationProperties; import feast.common.auth.credentials.CoreAuthenticationProperties; import feast.common.logging.config.LoggingProperties; -import feast.proto.core.StoreProto; +import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; +import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig; +import feast.storage.connectors.redis.retriever.RedisStoreConfig; +import io.lettuce.core.ReadFrom; +import java.time.Duration; import java.util.*; -import java.util.stream.Collectors; import javax.annotation.PostConstruct; -import javax.validation.ConstraintViolation; -import javax.validation.ConstraintViolationException; -import javax.validation.Validation; -import javax.validation.Validator; -import javax.validation.ValidatorFactory; +import javax.validation.*; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; import javax.validation.constraints.Positive; @@ -76,6 +72,26 @@ public FeastProperties() {} /* Feast Core port to connect to. */ @Positive private int coreGrpcPort; + private String registry; + + public String getRegistry() { + return registry; + } + + public void setRegistry(final String registry) { + this.registry = registry; + } + + private String transformationServiceEndpoint; + + public String getTransformationServiceEndpoint() { + return transformationServiceEndpoint; + } + + public void setTransformationServiceEndpoint(final String transformationServiceEndpoint) { + this.transformationServiceEndpoint = transformationServiceEndpoint; + } + private CoreAuthenticationProperties coreAuthentication; public CoreAuthenticationProperties getCoreAuthentication() { @@ -86,7 +102,6 @@ public void setCoreAuthentication(CoreAuthenticationProperties coreAuthenticatio this.coreAuthentication = coreAuthentication; } - /* Feast Core port to connect to. */ @Positive private int coreCacheRefreshInterval; private SecurityProperties security; @@ -146,9 +161,6 @@ public void setActiveStore(String activeStore) { */ private List stores = new ArrayList<>(); - /* Job Store properties to retain state of async jobs. */ - private JobStoreProperties jobStore; - /* Metric tracing properties. */ private TracingProperties tracing; @@ -259,8 +271,6 @@ public static class Store { private Map config = new HashMap<>(); - private List subscriptions = new ArrayList<>(); - /** * Gets name of this store. This is unique to this specific instance. * @@ -280,12 +290,12 @@ public void setName(String name) { } /** - * Gets the store type. Example are REDIS or BIGQUERY + * Gets the store type. Example are REDIS, REDIS_CLUSTER, BIGTABLE or CASSANDRA * * @return the store type as a String. */ - public String getType() { - return type; + public StoreType getType() { + return StoreType.valueOf(this.type); } /** @@ -297,64 +307,6 @@ public void setType(String type) { this.type = type; } - /** - * Converts this {@link Store} to a {@link StoreProto.Store} - * - * @return {@link StoreProto.Store} with configuration set - * @throws InvalidProtocolBufferException the invalid protocol buffer exception - * @throws JsonProcessingException the json processing exception - */ - public StoreProto.Store toProto() - throws InvalidProtocolBufferException, JsonProcessingException { - List subscriptions = getSubscriptions(); - List subscriptionProtos = - subscriptions.stream().map(Subscription::toProto).collect(Collectors.toList()); - - StoreProto.Store.Builder storeProtoBuilder = - StoreProto.Store.newBuilder() - .setName(name) - .setType(StoreProto.Store.StoreType.valueOf(type)) - .addAllSubscriptions(subscriptionProtos); - - ObjectMapper jsonWriter = new ObjectMapper(); - - // TODO: All of this logic should be moved to the store layer. Only a Map - // should be sent to a store and it should do its own validation. - switch (StoreProto.Store.StoreType.valueOf(type)) { - case REDIS_CLUSTER: - StoreProto.Store.RedisClusterConfig.Builder redisClusterConfig = - StoreProto.Store.RedisClusterConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisClusterConfig); - return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig.build()).build(); - case REDIS: - StoreProto.Store.RedisConfig.Builder redisConfig = - StoreProto.Store.RedisConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisConfig); - return storeProtoBuilder.setRedisConfig(redisConfig.build()).build(); - default: - throw new InvalidProtocolBufferException("Invalid store set"); - } - } - - /** - * Get the subscriptions to this specific store. The subscriptions indicate which feature sets a - * store subscribes to. - * - * @return List of subscriptions. - */ - public List getSubscriptions() { - return subscriptions; - } - - /** - * Sets the store specific configuration. See getSubscriptions() for more details. - * - * @param subscriptions the subscriptions list - */ - public void setSubscriptions(List subscriptions) { - this.subscriptions = subscriptions; - } - /** * Gets the configuration to this specific store. This is a map of strings. These options are * unique to the store. Please see protos/feast/core/Store.proto for the store specific @@ -366,6 +318,37 @@ public Map getConfig() { return config; } + public RedisClusterStoreConfig getRedisClusterConfig() { + return new RedisClusterStoreConfig( + this.config.get("connection_string"), + ReadFrom.valueOf(this.config.get("read_from")), + Duration.parse(this.config.get("timeout")), + Boolean.valueOf(this.config.getOrDefault("ssl", "false")), + this.config.getOrDefault("password", "")); + } + + public RedisStoreConfig getRedisConfig() { + return new RedisStoreConfig( + this.config.get("host"), + Integer.valueOf(this.config.get("port")), + Boolean.valueOf(this.config.getOrDefault("ssl", "false")), + this.config.getOrDefault("password", "")); + } + + public BigTableStoreConfig getBigtableConfig() { + return new BigTableStoreConfig( + this.config.get("project_id"), + this.config.get("instance_id"), + this.config.get("app_profile_id")); + } + + public CassandraStoreConfig getCassandraConfig() { + return new CassandraStoreConfig( + this.config.get("connection_string"), + this.config.get("data_center"), + this.config.get("keyspace")); + } + /** * Sets the store config. Please protos/feast/core/Store.proto for the specific options for each * store. @@ -375,129 +358,13 @@ public Map getConfig() { public void setConfig(Map config) { this.config = config; } - - /** - * The Subscription type. - * - *

Note: Please see protos/feast/core/CoreService.proto for details on how to subscribe to - * feature sets. - */ - public static class Subscription { - /** Feast project to subscribe to. */ - String project; - - /** Feature set to subscribe to. */ - String name; - - /** Feature set versions to subscribe to. */ - String version; - - /** Project/Feature set exclude flag to subscribe to. */ - boolean exclude; - - /** - * Gets Feast project subscribed to. - * - * @return the project string - */ - public String getProject() { - return project; - } - - /** - * Sets Feast project to subscribe to for this store. - * - * @param project the project - */ - public void setProject(String project) { - this.project = project; - } - - /** - * Gets the feature set name to subscribe to. - * - * @return the name - */ - public String getName() { - return name; - } - - /** - * Sets the feature set name to subscribe to. - * - * @param name the name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Gets the feature set version that is being subscribed to by this store. - * - * @return the version - */ - public String getVersion() { - return version; - } - - /** - * Sets the feature set version that is being subscribed to by this store. - * - * @param version the version - */ - public void setVersion(String version) { - this.version = version; - } - - /** - * Gets the exclude flag to subscribe to. - * - * @return the exclude flag - */ - public boolean getExclude() { - return exclude; - } - - /** - * Sets the exclude flag to subscribe to. - * - * @param exclude the exclude flag - */ - public void setExclude(boolean exclude) { - this.exclude = exclude; - } - - /** - * Convert this {@link Subscription} to a {@link StoreProto.Store.Subscription}. - * - * @return the store proto . store . subscription - */ - public StoreProto.Store.Subscription toProto() { - return StoreProto.Store.Subscription.newBuilder() - .setName(getName()) - .setProject(getProject()) - .setExclude(getExclude()) - .build(); - } - } } - /** - * Gets job store properties - * - * @return the job store properties - */ - public JobStoreProperties getJobStore() { - return jobStore; - } - - /** - * Set job store properties - * - * @param jobStore Job store properties to set - */ - public void setJobStore(JobStoreProperties jobStore) { - this.jobStore = jobStore; + public enum StoreType { + BIGTABLE, + CASSANDRA, + REDIS, + REDIS_CLUSTER; } /** @@ -527,57 +394,15 @@ public LoggingProperties getLogging() { return logging; } - /** Sets logging properties @@param logging the logging properties */ + /** + * Sets logging properties + * + * @param logging the logging properties + */ public void setLogging(LoggingProperties logging) { this.logging = logging; } - /** The type Job store properties. */ - public static class JobStoreProperties { - - /** Job Store Redis Host */ - private String redisHost; - - /** Job Store Redis Host */ - private int redisPort; - - /** - * Gets redis host. - * - * @return the redis host - */ - public String getRedisHost() { - return redisHost; - } - - /** - * Sets redis host. - * - * @param redisHost the redis host - */ - public void setRedisHost(String redisHost) { - this.redisHost = redisHost; - } - - /** - * Gets redis port. - * - * @return the redis port - */ - public int getRedisPort() { - return redisPort; - } - - /** - * Sets redis port. - * - * @param redisPort the redis port - */ - public void setRedisPort(int redisPort) { - this.redisPort = redisPort; - } - } - /** Trace metric collection properties */ public static class TracingProperties { diff --git a/serving/src/main/java/feast/serving/config/RegistryCondition.java b/serving/src/main/java/feast/serving/config/RegistryCondition.java new file mode 100644 index 0000000..621d124 --- /dev/null +++ b/serving/src/main/java/feast/serving/config/RegistryCondition.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from the Registry object. This is needed for versions of the feature store written by feast + * 0.10+. + */ +public class RegistryCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") != null; + } +} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 9ec35c3..ce2aabf 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -16,51 +16,169 @@ */ package feast.serving.config; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.protobuf.AbstractMessageLite; +import feast.serving.registry.LocalRegistryRepo; import feast.serving.service.OnlineServingServiceV2; +import feast.serving.service.OnlineTransformationService; import feast.serving.service.ServingServiceV2; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; +import feast.serving.specs.FeatureSpecRetriever; +import feast.serving.specs.RegistryFeatureSpecRetriever; import feast.storage.api.retriever.OnlineRetrieverV2; +import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever; +import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraOnlineRetriever; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.*; import io.opentracing.Tracer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; @Configuration public class ServingServiceConfigV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); + @Autowired private ApplicationContext context; + @Bean + @Lazy(true) + public BigtableDataClient bigtableClient(FeastProperties feastProperties) throws IOException { + BigTableStoreConfig config = feastProperties.getActiveStore().getBigtableConfig(); + String projectId = config.getProjectId(); + String instanceId = config.getInstanceId(); + + return BigtableDataClient.create( + BigtableDataSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setAppProfileId(config.getAppProfileId()) + .build()); + } + + @Bean + @Conditional(CoreCondition.class) public ServingServiceV2 servingServiceV2( - FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) - throws InvalidProtocolBufferException, JsonProcessingException { - ServingServiceV2 servingService = null; - FeastProperties.Store store = feastProperties.getActiveStore(); - StoreProto.Store.StoreType storeType = store.toProto().getType(); + FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) { + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); - switch (storeType) { + OnlineRetrieverV2 retrieverV2; + switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = - RedisClusterClient.create(store.toProto().getRedisClusterConfig()); - OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient); - servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer); + RedisClusterClient.create(store.getRedisClusterConfig()); + retrieverV2 = new OnlineRetriever(redisClusterClient, (AbstractMessageLite::toByteArray)); break; case REDIS: - RedisClientAdapter redisClient = RedisClient.create(store.toProto().getRedisConfig()); - OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient); - servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer); + RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); + retrieverV2 = new OnlineRetriever(redisClient, (AbstractMessageLite::toByteArray)); + break; + case BIGTABLE: + BigtableDataClient bigtableClient = context.getBean(BigtableDataClient.class); + retrieverV2 = new BigTableOnlineRetriever(bigtableClient); + break; + case CASSANDRA: + CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); + String connectionString = config.getConnectionString(); + String dataCenter = config.getDataCenter(); + String keySpace = config.getKeySpace(); + + List contactPoints = + Arrays.stream(connectionString.split(",")) + .map(String::trim) + .map(cs -> cs.split(":")) + .map( + hostPort -> { + int port = hostPort.length > 1 ? Integer.parseInt(hostPort[1]) : 9042; + return new InetSocketAddress(hostPort[0], port); + }) + .collect(Collectors.toList()); + + CqlSession session = + new CqlSessionBuilder() + .addContactPoints(contactPoints) + .withLocalDatacenter(dataCenter) + .withKeyspace(keySpace) + .build(); + retrieverV2 = new CassandraOnlineRetriever(session); break; - case UNRECOGNIZED: - case INVALID: - throw new IllegalArgumentException( + default: + throw new RuntimeException( + String.format("Unable to identify online store type: %s", store.getType())); + } + + final FeatureSpecRetriever featureSpecRetriever; + log.info("Created CoreFeatureSpecRetriever"); + featureSpecRetriever = new CoreFeatureSpecRetriever(specService); + + final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint(); + final OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); + + servingService = + new OnlineServingServiceV2( + retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); + + return servingService; + } + + @Bean + @Conditional(RegistryCondition.class) + public ServingServiceV2 registryBasedServingServiceV2( + FeastProperties feastProperties, Tracer tracer) { + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); + + OnlineRetrieverV2 retrieverV2; + // TODO: Support more store types, and potentially use a plugin model here. + switch (store.getType()) { + case REDIS_CLUSTER: + RedisClientAdapter redisClusterClient = + RedisClusterClient.create(store.getRedisClusterConfig()); + retrieverV2 = new OnlineRetriever(redisClusterClient, new EntityKeySerializerV2()); + break; + case REDIS: + RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); + log.info("Created EntityKeySerializerV2"); + retrieverV2 = new OnlineRetriever(redisClient, new EntityKeySerializerV2()); + break; + default: + throw new RuntimeException( String.format( - "Unsupported store type '%s' for store name '%s'", - store.getType(), store.getName())); + "Unable to identify online store type: %s for Regsitry Backed Serving Service", + store.getType())); } + final FeatureSpecRetriever featureSpecRetriever; + log.info("Created RegistryFeatureSpecRetriever"); + log.info("Working Directory = " + System.getProperty("user.dir")); + final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); + featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); + + final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint(); + final OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); + + servingService = + new OnlineServingServiceV2( + retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); + return servingService; } } diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java index b41a0f5..29d3bf0 100644 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java @@ -16,9 +16,6 @@ */ package feast.serving.config; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; import io.grpc.CallCredentials; @@ -29,15 +26,16 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; @Configuration public class SpecServiceConfig { private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); - private String feastCoreHost; - private int feastCorePort; - private int feastCachedSpecServiceRefreshInterval; + private final String feastCoreHost; + private final int feastCorePort; + private final int feastCachedSpecServiceRefreshInterval; @Autowired public SpecServiceConfig(FeastProperties feastProperties) { @@ -47,6 +45,7 @@ public SpecServiceConfig(FeastProperties feastProperties) { } @Bean + @Conditional(CoreCondition.class) public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( CachedSpecService cachedSpecStorage) { ScheduledExecutorService scheduledExecutorService = @@ -61,13 +60,11 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( } @Bean - public CachedSpecService specService( - FeastProperties feastProperties, ObjectProvider callCredentials) - throws InvalidProtocolBufferException, JsonProcessingException { + @Conditional(CoreCondition.class) + public CachedSpecService specService(ObjectProvider callCredentials) { CoreSpecService coreService = new CoreSpecService(feastCoreHost, feastCorePort, callCredentials); - StoreProto.Store storeProto = feastProperties.getActiveStore().toProto(); - CachedSpecService cachedSpecStorage = new CachedSpecService(coreService, storeProto); + CachedSpecService cachedSpecStorage = new CachedSpecService(coreService); try { cachedSpecStorage.populateCache(); } catch (Exception e) { diff --git a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java b/serving/src/main/java/feast/serving/config/WebSecurityConfig.java index f7b24a7..04d3f4b 100644 --- a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java +++ b/serving/src/main/java/feast/serving/config/WebSecurityConfig.java @@ -33,7 +33,7 @@ public class WebSecurityConfig extends WebSecurityConfigurerAdapter { * Allows for custom web security rules to be applied. * * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception + * @throws Exception exception */ @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java index 6615cf5..ef675d4 100644 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ b/serving/src/main/java/feast/serving/controller/HealthServiceController.java @@ -16,11 +16,9 @@ */ package feast.serving.controller; -import feast.proto.core.StoreProto.Store; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; import io.grpc.health.v1.HealthGrpc.HealthImplBase; import io.grpc.health.v1.HealthProto.HealthCheckRequest; import io.grpc.health.v1.HealthProto.HealthCheckResponse; @@ -33,12 +31,10 @@ @GrpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class HealthServiceController extends HealthImplBase { - private CachedSpecService specService; - private ServingServiceV2 servingService; + private final ServingServiceV2 servingService; @Autowired - public HealthServiceController(CachedSpecService specService, ServingServiceV2 servingService) { - this.specService = specService; + public HealthServiceController(final ServingServiceV2 servingService) { this.servingService = servingService; } @@ -48,10 +44,9 @@ public void check( // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g. // if it's online service check that it the service can retrieve dummy/random // feature table. - // Implement similary for batch service. + // Implement similarly for batch service. try { - Store store = specService.getStore(); servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance()); responseObserver.onNext( HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 531be39..81bbfd0 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -25,6 +25,7 @@ import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.serving.config.FeastProperties; import feast.serving.exception.SpecRetrievalException; +import feast.serving.interceptors.GrpcMonitoringContext; import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingServiceV2; import feast.serving.util.RequestHelper; @@ -86,6 +87,9 @@ public void getOnlineFeaturesV2( // project set at root level overrides the project set at feature table level this.authorizationService.authorizeRequest( SecurityContextHolder.getContext(), request.getProject()); + + // update monitoring context + GrpcMonitoringContext.getInstance().setProject(request.getProject()); } RequestHelper.validateOnlineRequest(request); Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java new file mode 100644 index 0000000..48d8d76 --- /dev/null +++ b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.interceptors; + +import java.util.Optional; + +public class GrpcMonitoringContext { + private static GrpcMonitoringContext INSTANCE; + + final ThreadLocal project = new ThreadLocal(); + + private GrpcMonitoringContext() {} + + public static GrpcMonitoringContext getInstance() { + if (INSTANCE == null) { + INSTANCE = new GrpcMonitoringContext(); + } + + return INSTANCE; + } + + public void setProject(String name) { + this.project.set(name); + } + + public Optional getProject() { + return Optional.ofNullable(this.project.get()); + } + + public void clearProject() { + this.project.set(null); + } +} diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java index bc7ed89..735f8c5 100644 --- a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java +++ b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java @@ -24,6 +24,7 @@ import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.Status; +import java.util.Optional; /** * GrpcMonitoringInterceptor intercepts GRPC calls to provide request latency histogram metrics in @@ -39,12 +40,16 @@ public Listener interceptCall( String fullMethodName = call.getMethodDescriptor().getFullMethodName(); String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); + GrpcMonitoringContext.getInstance().clearProject(); + return next.startCall( new SimpleForwardingServerCall(call) { @Override public void close(Status status, Metadata trailers) { + Optional projectName = GrpcMonitoringContext.getInstance().getProject(); + Metrics.requestLatency - .labels(methodName) + .labels(methodName, projectName.orElse("")) .observe((System.currentTimeMillis() - startCallMillis) / 1000f); Metrics.grpcRequestCount.labels(methodName, status.getCode().name()).inc(); super.close(status, trailers); diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java new file mode 100644 index 0000000..4178541 --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class LocalRegistryRepo implements RegistryRepository { + private final RegistryProto.Registry registry; + private Map featureViewNameToSpec; + private Map + onDemandFeatureViewNameToSpec; + + public LocalRegistryRepo(Path localRegistryPath) { + if (!localRegistryPath.toFile().exists()) { + throw new RuntimeException( + String.format("Local registry not found at path %s", localRegistryPath)); + } + try { + final byte[] registryContents = Files.readAllBytes(localRegistryPath); + this.registry = RegistryProto.Registry.parseFrom(registryContents); + } catch (final Exception e) { + throw new RuntimeException(e); + } + + final RegistryProto.Registry registry = this.getRegistry(); + List featureViewSpecs = + registry.getFeatureViewsList().stream() + .map(fv -> fv.getSpec()) + .collect(Collectors.toList()); + featureViewNameToSpec = + featureViewSpecs.stream() + .collect( + Collectors.toMap(FeatureViewProto.FeatureViewSpec::getName, Function.identity())); + List onDemandFeatureViewSpecs = + registry.getOnDemandFeatureViewsList().stream() + .map(odfv -> odfv.getSpec()) + .collect(Collectors.toList()); + onDemandFeatureViewNameToSpec = + onDemandFeatureViewSpecs.stream() + .collect( + Collectors.toMap( + OnDemandFeatureViewProto.OnDemandFeatureViewSpec::getName, + Function.identity())); + } + + @Override + public RegistryProto.Registry getRegistry() { + return this.registry; + } + + @Override + public FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + String featureViewName = featureReference.getFeatureTable(); + if (featureViewNameToSpec.containsKey(featureViewName)) { + return featureViewNameToSpec.get(featureViewName); + } + throw new SpecRetrievalException( + String.format("Unable to find feature view with name: %s", featureViewName)); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final FeatureViewProto.FeatureViewSpec spec = + this.getFeatureViewSpec(projectName, featureReference); + for (final FeatureProto.FeatureSpecV2 featureSpec : spec.getFeaturesList()) { + if (featureSpec.getName().equals(featureReference.getName())) { + return featureSpec; + } + } + + throw new SpecRetrievalException( + String.format( + "Unable to find feature with name: %s in feature view: %s", + featureReference.getName(), featureReference.getFeatureTable())); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + if (onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName)) { + return onDemandFeatureViewNameToSpec.get(onDemandFeatureViewName); + } + throw new SpecRetrievalException( + String.format( + "Unable to find on demand feature view with name: %s", onDemandFeatureViewName)); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + return onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName); + } +} diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java new file mode 100644 index 0000000..21a2183 --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/RegistryRepository.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; + +/** + * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry + * object. This approach is needed for a feature store created using feast 0.10+. + */ +public interface RegistryRepository { + RegistryProto.Registry getRegistry(); + + FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 70dd6f7..2cd810c 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -26,54 +26,47 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; import feast.proto.types.ValueProto; import feast.serving.exception.SpecRetrievalException; -import feast.serving.specs.CachedSpecService; +import feast.serving.specs.FeatureSpecRetriever; import feast.serving.util.Metrics; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.OnlineRetrieverV2; import io.grpc.Status; import io.opentracing.Span; import io.opentracing.Tracer; -import java.util.*; +import java.io.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; public class OnlineServingServiceV2 implements ServingServiceV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class); - private final CachedSpecService specService; private final Tracer tracer; private final OnlineRetrieverV2 retriever; - - private static final HashMap - TYPE_TO_VAL_CASE = - new HashMap<>() { - { - put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); - put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); - put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); - put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); - put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); - put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); - put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); - put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); - put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); - put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); - put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); - put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); - put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); - put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); - } - }; + private final FeatureSpecRetriever featureSpecRetriever; + private final OnlineTransformationService onlineTransformationService; public OnlineServingServiceV2( - OnlineRetrieverV2 retriever, CachedSpecService specService, Tracer tracer) { + OnlineRetrieverV2 retriever, + Tracer tracer, + FeatureSpecRetriever featureSpecRetriever, + OnlineTransformationService onlineTransformationService) { this.retriever = retriever; - this.specService = specService; this.tracer = tracer; + this.featureSpecRetriever = featureSpecRetriever; + this.onlineTransformationService = onlineTransformationService; } /** {@inheritDoc} */ @@ -87,41 +80,62 @@ public GetFeastServingInfoResponse getFeastServingInfo( @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 request) { - String projectName = request.getProject(); - List featureReferences = request.getFeaturesList(); - // Autofill default project if project is not specified + String projectName = request.getProject(); if (projectName.isEmpty()) { projectName = "default"; } - List entityRows = request.getEntityRowsList(); + // Split all feature references into non-ODFV (e.g. batch and stream) references and ODFV. + List allFeatureReferences = request.getFeaturesList(); + List featureReferences = + allFeatureReferences.stream() + .filter(r -> !this.featureSpecRetriever.isOnDemandFeatureReference(r)) + .collect(Collectors.toList()); + List onDemandFeatureReferences = + allFeatureReferences.stream() + .filter(r -> this.featureSpecRetriever.isOnDemandFeatureReference(r)) + .collect(Collectors.toList()); + + // Get the set of request data feature names and feature inputs from the ODFV references. + Pair, List> pair = + this.onlineTransformationService.extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + onDemandFeatureReferences, projectName); + Set requestDataFeatureNames = pair.getLeft(); + List onDemandFeatureInputs = pair.getRight(); + + // Add on demand feature inputs to list of feature references to retrieve. + Set addedFeatureReferences = new HashSet(); + for (FeatureReferenceV2 onDemandFeatureInput : onDemandFeatureInputs) { + if (!featureReferences.contains(onDemandFeatureInput)) { + featureReferences.add(onDemandFeatureInput); + addedFeatureReferences.add(onDemandFeatureInput); + } + } + + // Separate entity rows into entity data and request feature data. + Pair, Map>> + entityRowsAndRequestDataFeatures = + this.onlineTransformationService.separateEntityRows(requestDataFeatureNames, request); + List entityRows = + entityRowsAndRequestDataFeatures.getLeft(); + Map> requestDataFeatures = + entityRowsAndRequestDataFeatures.getRight(); + // TODO: error checking on lengths of lists in entityRows and requestDataFeatures + + // Extract values and statuses to be used later in constructing FieldValues for the response. + // The online features retrieved will augment these two data structures. List> values = entityRows.stream().map(r -> new HashMap<>(r.getFieldsMap())).collect(Collectors.toList()); List> statuses = entityRows.stream() - .map(r -> getMetadataMap(r.getFieldsMap(), false, false)) + .map( + r -> + r.getFieldsMap().entrySet().stream() + .map(entry -> Pair.of(entry.getKey(), getMetadata(entry.getValue(), false))) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight))) .collect(Collectors.toList()); - Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.setTag("entities", entityRows.size()); - storageRetrievalSpan.setTag("features", featureReferences.size()); - } - List> entityRowsFeatures = - retriever.getOnlineFeatures(projectName, entityRows, featureReferences); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.finish(); - } - - if (entityRowsFeatures.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } - String finalProjectName = projectName; Map featureMaxAges = featureReferences.stream() @@ -129,7 +143,12 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re .collect( Collectors.toMap( Function.identity(), - ref -> specService.getFeatureTableSpec(finalProjectName, ref).getMaxAge())); + ref -> this.featureSpecRetriever.getMaxAge(finalProjectName, ref))); + List entityNames = + featureReferences.stream() + .map(ref -> this.featureSpecRetriever.getEntitiesList(finalProjectName, ref)) + .findFirst() + .get(); Map featureValueTypes = featureReferences.stream() @@ -139,12 +158,33 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re Function.identity(), ref -> { try { - return specService.getFeatureSpec(finalProjectName, ref).getValueType(); + return this.featureSpecRetriever + .getFeatureSpec(finalProjectName, ref) + .getValueType(); } catch (SpecRetrievalException e) { return ValueProto.ValueType.Enum.INVALID; } })); + Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.setTag("entities", entityRows.size()); + storageRetrievalSpan.setTag("features", featureReferences.size()); + } + List> entityRowsFeatures = + retriever.getOnlineFeatures(projectName, entityRows, featureReferences, entityNames); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.finish(); + } + + if (entityRowsFeatures.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); + } + Span postProcessingSpan = tracer.buildSpan("postProcessing").start(); for (int i = 0; i < entityRows.size(); i++) { @@ -161,44 +201,35 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re if (featureReferenceFeatureMap.containsKey(featureReference)) { Feature feature = featureReferenceFeatureMap.get(featureReference); - ValueProto.Value.ValCase valueCase = feature.getFeatureValue().getValCase(); + ValueProto.Value value = + feature.getFeatureValue(featureValueTypes.get(feature.getFeatureReference())); - boolean isMatchingFeatureSpec = - checkSameFeatureSpec(featureValueTypes.get(feature.getFeatureReference()), valueCase); - boolean isOutsideMaxAge = + Boolean isOutsideMaxAge = checkOutsideMaxAge( feature, entityRow, featureMaxAges.get(feature.getFeatureReference())); - Map valueMap = - unpackValueMap(feature, isOutsideMaxAge, isMatchingFeatureSpec); - rowValues.putAll(valueMap); - - // Generate metadata for feature values and merge into entityFieldsMap - Map statusMap = - getMetadataMap(valueMap, !isMatchingFeatureSpec, isOutsideMaxAge); - rowStatuses.putAll(statusMap); + if (!isOutsideMaxAge && value != null) { + rowValues.put(FeatureV2.getFeatureStringRef(feature.getFeatureReference()), value); + } else { + rowValues.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), + ValueProto.Value.newBuilder().build()); + } - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); + rowStatuses.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), + getMetadata(value, isOutsideMaxAge)); } else { - Map valueMap = - new HashMap<>() { - { - put( - FeatureV2.getFeatureStringRef(featureReference), - ValueProto.Value.newBuilder().build()); - } - }; - rowValues.putAll(valueMap); - - Map statusMap = - getMetadataMap(valueMap, true, false); - rowStatuses.putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); + rowValues.put( + FeatureV2.getFeatureStringRef(featureReference), + ValueProto.Value.newBuilder().build()); + + rowStatuses.put( + FeatureV2.getFeatureStringRef(featureReference), getMetadata(null, false)); } } + // Populate metrics/log request + populateCountMetrics(rowStatuses, projectName); } if (postProcessingSpan != null) { @@ -208,6 +239,71 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re populateHistogramMetrics(entityRows, featureReferences, projectName); populateFeatureCountMetrics(featureReferences, projectName); + // Handle ODFVs. For each ODFV reference, we send a TransformFeaturesRequest to the FTS. + // The request should contain the entity data, the retrieved features, and the request data. + if (!onDemandFeatureReferences.isEmpty()) { + // Augment values, which contains the entity data and retrieved features, with the request + // data. Also augment statuses. + for (int i = 0; i < values.size(); i++) { + Map rowValues = values.get(i); + Map rowStatuses = statuses.get(i); + + for (Map.Entry> entry : requestDataFeatures.entrySet()) { + String key = entry.getKey(); + List fieldValues = entry.getValue(); + rowValues.put(key, fieldValues.get(i)); + rowStatuses.put(key, GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + } + + // Serialize the augmented values. + ValueType transformationInput = + this.onlineTransformationService.serializeValuesIntoArrowIPC(values); + + // Send out requests to the FTS and process the responses. + Set onDemandFeatureStringReferences = + onDemandFeatureReferences.stream() + .map(r -> FeatureV2.getFeatureStringRef(r)) + .collect(Collectors.toSet()); + for (FeatureReferenceV2 featureReference : onDemandFeatureReferences) { + String onDemandFeatureViewName = featureReference.getFeatureTable(); + TransformFeaturesRequest transformFeaturesRequest = + TransformFeaturesRequest.newBuilder() + .setOnDemandFeatureViewName(onDemandFeatureViewName) + .setProject(projectName) + .setTransformationInput(transformationInput) + .build(); + + TransformFeaturesResponse transformFeaturesResponse = + this.onlineTransformationService.transformFeatures(transformFeaturesRequest); + + this.onlineTransformationService.processTransformFeaturesResponse( + transformFeaturesResponse, + onDemandFeatureViewName, + onDemandFeatureStringReferences, + values, + statuses); + } + + // Remove all features that were added as inputs for ODFVs. + Set addedFeatureStringReferences = + addedFeatureReferences.stream() + .map(r -> FeatureV2.getFeatureStringRef(r)) + .collect(Collectors.toSet()); + for (int i = 0; i < values.size(); i++) { + Map rowValues = values.get(i); + Map rowStatuses = statuses.get(i); + List keysToRemove = + rowValues.keySet().stream() + .filter(k -> addedFeatureStringReferences.contains(k)) + .collect(Collectors.toList()); + for (String key : keysToRemove) { + rowValues.remove(key); + rowStatuses.remove(key); + } + } + } + // Build response field values from entityValuesMap and entityStatusesMap // Response field values should be in the same order as the entityRows provided by the user. List fieldValuesList = @@ -219,20 +315,8 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re .putAllStatuses(statuses.get(entityRowIdx)) .build()) .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); - } - private boolean checkSameFeatureSpec( - ValueProto.ValueType.Enum valueTypeEnum, ValueProto.Value.ValCase valueCase) { - if (valueTypeEnum.equals(ValueProto.ValueType.Enum.INVALID)) { - return false; - } - - if (valueCase.equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return true; - } - - return TYPE_TO_VAL_CASE.get(valueTypeEnum).equals(valueCase); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } private static Map getFeatureRefFeatureMap(List features) { @@ -243,47 +327,23 @@ private static Map getFeatureRefFeatureMap(List getMetadataMap( - Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { - return valueMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - es -> { - ValueProto.Value fieldValue = es.getValue(); - if (isNotFound) { - return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND; - } else if (isOutsideMaxAge) { - return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE; - } else if (fieldValue.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE; - } - return GetOnlineFeaturesResponse.FieldStatus.PRESENT; - })); - } - - private static Map unpackValueMap( - Feature feature, boolean isOutsideMaxAge, boolean isMatchingFeatureSpec) { - Map valueMap = new HashMap<>(); - - if (!isOutsideMaxAge && isMatchingFeatureSpec) { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), feature.getFeatureValue()); - } else { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), - ValueProto.Value.newBuilder().build()); + private static GetOnlineFeaturesResponse.FieldStatus getMetadata( + ValueProto.Value value, boolean isOutsideMaxAge) { + + if (value == null) { + return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND; + } else if (isOutsideMaxAge) { + return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE; + } else if (value.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { + return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE; } - - return valueMap; + return GetOnlineFeaturesResponse.FieldStatus.PRESENT; } /** diff --git a/serving/src/main/java/feast/serving/service/OnlineTransformationService.java b/serving/src/main/java/feast/serving/service/OnlineTransformationService.java new file mode 100644 index 0000000..541fe46 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/OnlineTransformationService.java @@ -0,0 +1,412 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import com.google.protobuf.ByteString; +import feast.common.models.FeatureV2; +import feast.proto.core.DataSourceProto; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; +import feast.proto.serving.TransformationServiceGrpc; +import feast.proto.types.ValueProto; +import feast.serving.specs.FeatureSpecRetriever; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Status; +import java.io.IOException; +import java.nio.channels.Channels; +import java.util.*; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tomcat.util.http.fileupload.ByteArrayOutputStream; +import org.slf4j.Logger; + +public class OnlineTransformationService implements TransformationService { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(OnlineTransformationService.class); + private final TransformationServiceGrpc.TransformationServiceBlockingStub stub; + private final FeatureSpecRetriever featureSpecRetriever; + static final int INT64_BITWIDTH = 64; + static final int INT32_BITWIDTH = 32; + + public OnlineTransformationService( + String transformationServiceEndpoint, FeatureSpecRetriever featureSpecRetriever) { + if (transformationServiceEndpoint != null) { + final ManagedChannel channel = + ManagedChannelBuilder.forTarget(transformationServiceEndpoint).usePlaintext().build(); + this.stub = TransformationServiceGrpc.newBlockingStub(channel); + } else { + this.stub = null; + } + this.featureSpecRetriever = featureSpecRetriever; + } + + /** {@inheritDoc} */ + @Override + public TransformFeaturesResponse transformFeatures( + TransformFeaturesRequest transformFeaturesRequest) { + return this.stub.transformFeatures(transformFeaturesRequest); + } + + /** {@inheritDoc} */ + @Override + public Pair, List> + extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + List onDemandFeatureReferences, String projectName) { + Set requestDataFeatureNames = new HashSet(); + List onDemandFeatureInputs = + new ArrayList(); + for (ServingAPIProto.FeatureReferenceV2 featureReference : onDemandFeatureReferences) { + OnDemandFeatureViewProto.OnDemandFeatureViewSpec onDemandFeatureViewSpec = + this.featureSpecRetriever.getOnDemandFeatureViewSpec(projectName, featureReference); + Map inputs = + onDemandFeatureViewSpec.getInputsMap(); + + for (OnDemandFeatureViewProto.OnDemandInput input : inputs.values()) { + OnDemandFeatureViewProto.OnDemandInput.InputCase inputCase = input.getInputCase(); + switch (inputCase) { + case REQUEST_DATA_SOURCE: + DataSourceProto.DataSource requestDataSource = input.getRequestDataSource(); + DataSourceProto.DataSource.RequestDataOptions requestDataOptions = + requestDataSource.getRequestDataOptions(); + Set requestDataNames = requestDataOptions.getSchemaMap().keySet(); + requestDataFeatureNames.addAll(requestDataNames); + break; + case FEATURE_VIEW: + FeatureViewProto.FeatureView featureView = input.getFeatureView(); + FeatureViewProto.FeatureViewSpec featureViewSpec = featureView.getSpec(); + String featureViewName = featureViewSpec.getName(); + for (FeatureProto.FeatureSpecV2 featureSpec : featureViewSpec.getFeaturesList()) { + String featureName = featureSpec.getName(); + ServingAPIProto.FeatureReferenceV2 onDemandFeatureInput = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable(featureViewName) + .setName(featureName) + .build(); + onDemandFeatureInputs.add(onDemandFeatureInput); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "OnDemandInput proto input field has an unexpected type: " + inputCase) + .asRuntimeException(); + } + } + } + Pair, List> pair = + new ImmutablePair, List>( + requestDataFeatureNames, onDemandFeatureInputs); + return pair; + } + + /** {@inheritDoc} */ + public Pair< + List, + Map>> + separateEntityRows( + Set requestDataFeatureNames, ServingAPIProto.GetOnlineFeaturesRequestV2 request) { + // Separate entity rows into entity data and request feature data. + List entityRows = + new ArrayList(); + Map> requestDataFeatures = + new HashMap>(); + + for (ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow : + request.getEntityRowsList()) { + Map fieldsMap = new HashMap(); + + for (Map.Entry entry : entityRow.getFieldsMap().entrySet()) { + String key = entry.getKey(); + ValueProto.Value value = entry.getValue(); + + if (requestDataFeatureNames.contains(key)) { + if (!requestDataFeatures.containsKey(key)) { + requestDataFeatures.put(key, new ArrayList()); + } + requestDataFeatures.get(key).add(value); + } else { + fieldsMap.put(key, value); + } + } + + // Construct new entity row containing the extracted entity data, if necessary. + if (!fieldsMap.isEmpty()) { + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow newEntityRow = + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(entityRow.getTimestamp()) + .putAllFields(fieldsMap) + .build(); + entityRows.add(newEntityRow); + } + } + + Pair< + List, + Map>> + pair = + new ImmutablePair< + List, + Map>>(entityRows, requestDataFeatures); + return pair; + } + + /** {@inheritDoc} */ + public void processTransformFeaturesResponse( + feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse + transformFeaturesResponse, + String onDemandFeatureViewName, + Set onDemandFeatureStringReferences, + List> values, + List> statuses) { + try { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ArrowFileReader reader = + new ArrowFileReader( + new ByteArrayReadableSeekableByteChannel( + transformFeaturesResponse + .getTransformationOutput() + .getArrowValue() + .toByteArray()), + allocator); + reader.loadNextBatch(); + VectorSchemaRoot readBatch = reader.getVectorSchemaRoot(); + Schema responseSchema = readBatch.getSchema(); + List responseFields = responseSchema.getFields(); + + for (Field field : responseFields) { + String columnName = field.getName(); + String fullFeatureName = onDemandFeatureViewName + ":" + columnName; + ArrowType columnType = field.getType(); + + // The response will contain all features for the specified ODFV, so we + // skip the features that were not requested. + if (!onDemandFeatureStringReferences.contains(fullFeatureName)) { + continue; + } + + FieldVector fieldVector = readBatch.getVector(field); + int valueCount = fieldVector.getValueCount(); + + // TODO: support all Feast types + // TODO: clean up the switch statement + if (columnType instanceof ArrowType.Int) { + int bitWidth = ((ArrowType.Int) columnType).getBitWidth(); + switch (bitWidth) { + case INT64_BITWIDTH: + for (int i = 0; i < valueCount; i++) { + long int64Value = ((BigIntVector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setInt64Val(int64Value).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + case INT32_BITWIDTH: + for (int i = 0; i < valueCount; i++) { + int intValue = ((IntVector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setInt32Val(intValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " is of type ArrowType.Int but has bitWidth " + + bitWidth + + " which cannot be handled.") + .asRuntimeException(); + } + } else if (columnType instanceof ArrowType.FloatingPoint) { + FloatingPointPrecision precision = ((ArrowType.FloatingPoint) columnType).getPrecision(); + switch (precision) { + case DOUBLE: + for (int i = 0; i < valueCount; i++) { + double doubleValue = ((Float8Vector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setDoubleVal(doubleValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + case SINGLE: + for (int i = 0; i < valueCount; i++) { + float floatValue = ((Float4Vector) fieldVector).get(i); + Map rowValues = values.get(i); + Map rowStatuses = + statuses.get(i); + ValueProto.Value value = + ValueProto.Value.newBuilder().setFloatVal(floatValue).build(); + rowValues.put(fullFeatureName, value); + rowStatuses.put( + fullFeatureName, ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT); + } + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " is of type ArrowType.FloatingPoint but has precision " + + precision + + " which cannot be handled.") + .asRuntimeException(); + } + } + } + } catch (IOException e) { + log.info(e.toString()); + throw Status.INTERNAL + .withDescription( + "Unable to correctly process transform features response: " + e.toString()) + .asRuntimeException(); + } + } + + /** {@inheritDoc} */ + public ValueType serializeValuesIntoArrowIPC(List> values) { + // In order to be serialized correctly, the data must be packaged in a VectorSchemaRoot. + // We first construct all the columns. + Map columnNameToColumn = new HashMap(); + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + Map firstAugmentedRowValues = values.get(0); + for (Map.Entry entry : firstAugmentedRowValues.entrySet()) { + // The Python FTS does not expect full feature names, so we extract the feature name. + String columnName = FeatureV2.getFeatureName(entry.getKey()); + ValueProto.Value.ValCase valCase = entry.getValue().getValCase(); + FieldVector column; + // TODO: support all Feast types + switch (valCase) { + case INT32_VAL: + column = new IntVector(columnName, allocator); + break; + case INT64_VAL: + column = new BigIntVector(columnName, allocator); + break; + case DOUBLE_VAL: + column = new Float8Vector(columnName, allocator); + break; + case FLOAT_VAL: + column = new Float4Vector(columnName, allocator); + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + columnName + " has a type that is currently not handled: " + valCase) + .asRuntimeException(); + } + column.allocateNew(); + columnNameToColumn.put(columnName, column); + } + + // Add the data, row by row. + for (int i = 0; i < values.size(); i++) { + Map augmentedRowValues = values.get(i); + + for (Map.Entry entry : augmentedRowValues.entrySet()) { + String columnName = FeatureV2.getFeatureName(entry.getKey()); + ValueProto.Value value = entry.getValue(); + ValueProto.Value.ValCase valCase = value.getValCase(); + FieldVector column = columnNameToColumn.get(columnName); + // TODO: support all Feast types + switch (valCase) { + case INT32_VAL: + ((IntVector) column).setSafe(i, value.getInt32Val()); + break; + case INT64_VAL: + ((BigIntVector) column).setSafe(i, value.getInt64Val()); + break; + case DOUBLE_VAL: + ((Float8Vector) column).setSafe(i, value.getDoubleVal()); + break; + case FLOAT_VAL: + ((Float4Vector) column).setSafe(i, value.getFloatVal()); + break; + default: + throw Status.INTERNAL + .withDescription( + "Column " + + columnName + + " has a type that is currently not handled: " + + valCase) + .asRuntimeException(); + } + } + } + + // Construct the VectorSchemaRoot. + List columnFields = new ArrayList(); + List columns = new ArrayList(); + for (FieldVector column : columnNameToColumn.values()) { + column.setValueCount(values.size()); + columnFields.add(column.getField()); + columns.add(column); + } + VectorSchemaRoot schemaRoot = new VectorSchemaRoot(columnFields, columns); + + // Serialize the VectorSchemaRoot into Arrow IPC format. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowFileWriter writer = new ArrowFileWriter(schemaRoot, null, Channels.newChannel(out)); + try { + writer.start(); + writer.writeBatch(); + writer.end(); + } catch (IOException e) { + log.info(e.toString()); + throw Status.INTERNAL + .withDescription( + "ArrowFileWriter could not write properly; failed with error: " + e.toString()) + .asRuntimeException(); + } + byte[] byteData = out.toByteArray(); + ByteString inputData = ByteString.copyFrom(byteData); + ValueType transformationInput = ValueType.newBuilder().setArrowValue(inputData).build(); + return transformationInput; + } +} diff --git a/serving/src/main/java/feast/serving/service/TransformationService.java b/serving/src/main/java/feast/serving/service/TransformationService.java new file mode 100644 index 0000000..caa5279 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/TransformationService.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesRequest; +import feast.proto.serving.TransformationServiceAPIProto.TransformFeaturesResponse; +import feast.proto.serving.TransformationServiceAPIProto.ValueType; +import feast.proto.types.ValueProto; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; + +public interface TransformationService { + /** + * Apply on demand transformations for the specified ODFVs. + * + * @param transformFeaturesRequest proto containing the ODFV references and necessary data + * @return a proto object containing the response + */ + TransformFeaturesResponse transformFeatures(TransformFeaturesRequest transformFeaturesRequest); + + /** + * Extract the set of request data feature names and the list of on demand feature inputs from a + * list of ODFV references. + * + * @param onDemandFeatureReferences list of ODFV references to be parsed + * @param projectName project name + * @return a pair containing the set of request data feature names and list of on demand feature + * inputs + */ + Pair, List> + extractRequestDataFeatureNamesAndOnDemandFeatureInputs( + List onDemandFeatureReferences, String projectName); + + /** + * Separate the entity rows of a request into entity data and request feature data. + * + * @param requestDataFeatureNames set of feature names for the request data + * @param request the GetOnlineFeaturesRequestV2 containing the entity rows + * @return a pair containing the set of request data feature names and list of on demand feature + * inputs + */ + Pair, Map>> + separateEntityRows(Set requestDataFeatureNames, GetOnlineFeaturesRequestV2 request); + + /** + * Process a response from the feature transformation server by augmenting the given lists of + * field maps and status maps with the correct fields from the response. + * + * @param transformFeaturesResponse response to be processed + * @param onDemandFeatureViewName name of ODFV to which the response corresponds + * @param onDemandFeatureStringReferences set of all ODFV references that should be kept + * @param values list of field maps to be augmented with additional fields from the response + * @param statuses list of status maps to be augmented + */ + void processTransformFeaturesResponse( + TransformFeaturesResponse transformFeaturesResponse, + String onDemandFeatureViewName, + Set onDemandFeatureStringReferences, + List> values, + List> statuses); + + /** + * Serialize data into Arrow IPC format, to be sent to the Python feature transformation server. + * + * @param values list of field maps to be serialized + * @return the data packaged into a ValueType proto object + */ + ValueType serializeValuesIntoArrowIPC(List> values); +} diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java index f54e08f..440b224 100644 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CachedSpecService.java @@ -26,8 +26,6 @@ import feast.proto.core.FeatureProto; import feast.proto.core.FeatureTableProto.FeatureTable; import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store; import feast.proto.serving.ServingAPIProto; import feast.serving.exception.SpecRetrievalException; import io.grpc.StatusRuntimeException; @@ -47,7 +45,6 @@ public class CachedSpecService { private static final String DEFAULT_PROJECT_NAME = "default"; private final CoreSpecService coreService; - private Store store; private static Gauge cacheLastUpdated = Gauge.build() @@ -69,9 +66,8 @@ public class CachedSpecService { ImmutablePair, FeatureProto.FeatureSpecV2> featureCache; - public CachedSpecService(CoreSpecService coreService, StoreProto.Store store) { + public CachedSpecService(CoreSpecService coreService) { this.coreService = coreService; - this.store = coreService.registerStore(store); CacheLoader, FeatureTableSpec> featureTableCacheLoader = CacheLoader.from(k -> retrieveSingleFeatureTable(k.getLeft(), k.getRight())); @@ -85,15 +81,6 @@ public CachedSpecService(CoreSpecService coreService, StoreProto.Store store) { featureCache = CacheBuilder.newBuilder().build(featureCacheLoader); } - /** - * Get the current store configuration. - * - * @return StoreProto.Store store configuration for this serving instance - */ - public Store getStore() { - return this.store; - } - /** * Reload the store configuration from the given config path, then retrieve the necessary specs * from core to preload the cache. diff --git a/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java new file mode 100644 index 0000000..2a88659 --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public class CoreFeatureSpecRetriever implements FeatureSpecRetriever { + private final CachedSpecService specService; + + public CoreFeatureSpecRetriever(CachedSpecService specService) { + this.specService = specService; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getMaxAge(); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getEntitiesList(); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureSpec(projectName, featureReference); + } + + @Override + public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + throw new UnsupportedOperationException( + String.format("Feast Core does not support getting feature view specs.")); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + throw new UnsupportedOperationException( + String.format("Feast Core does not support on demand feature views.")); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + return false; + } +} diff --git a/serving/src/main/java/feast/serving/specs/CoreSpecService.java b/serving/src/main/java/feast/serving/specs/CoreSpecService.java index 5429d22..eee50d8 100644 --- a/serving/src/main/java/feast/serving/specs/CoreSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CoreSpecService.java @@ -24,7 +24,6 @@ import feast.proto.core.CoreServiceProto.ListProjectsResponse; import feast.proto.core.CoreServiceProto.UpdateStoreRequest; import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import feast.proto.core.StoreProto.Store; import io.grpc.CallCredentials; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -53,26 +52,6 @@ public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) { return blockingStub.updateStore(updateStoreRequest); } - /** - * Register the given store entry in Feast Core. If store already exists in Feast Core, updates - * the store entry in feast core. - * - * @param store entry to register/update in Feast Core. - * @return The register/updated store entry - */ - public Store registerStore(Store store) { - UpdateStoreRequest request = UpdateStoreRequest.newBuilder().setStore(store).build(); - try { - UpdateStoreResponse updateStoreResponse = this.updateStore(request); - if (!updateStoreResponse.getStore().equals(store)) { - throw new RuntimeException("Core store config not matching current store config"); - } - return updateStoreResponse.getStore(); - } catch (Exception e) { - throw new RuntimeException("Unable to update store configuration", e); - } - } - public ListProjectsResponse listProjects(ListProjectsRequest listProjectsRequest) { return blockingStub.listProjects(listProjectsRequest); } diff --git a/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java new file mode 100644 index 0000000..57e931c --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public interface FeatureSpecRetriever { + + Duration getMaxAge(String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java new file mode 100644 index 0000000..0cd851e --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.OnDemandFeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import feast.serving.registry.RegistryRepository; +import java.util.List; + +public class RegistryFeatureSpecRetriever implements FeatureSpecRetriever { + private final RegistryRepository registryRepository; + + public RegistryFeatureSpecRetriever(RegistryRepository registryRepository) { + this.registryRepository = registryRepository; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getTtl(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getEntitiesList(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getFeatureSpec(projectName, featureReference); + } + + @Override + public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getFeatureViewSpec(projectName, featureReference); + } + + @Override + public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getOnDemandFeatureViewSpec(projectName, featureReference); + } + + @Override + public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.isOnDemandFeatureReference(featureReference); + } +} diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 90b9493..dca2b5e 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -26,7 +26,7 @@ public class Metrics { .name("request_latency_seconds") .subsystem("feast_serving") .help("Request latency in seconds") - .labelNames("method") + .labelNames("method", "project") .register(); public static final Histogram requestEntityCountDistribution = diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 288ec7e..3e4e07b 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -3,7 +3,7 @@ feast: # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: ${FEAST_CORE_HOST:localhost} core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - + core-authentication: enabled: false # should be set to true if authentication is enabled on core. provider: google # can be set to `oauth` or `google` @@ -41,26 +41,31 @@ feast: stores: # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options - name: online # Name of the store (referenced by active_store) - type: REDIS # Type of the store. REDIS, REDIS_CLUSTER, BIGQUERY are available options + type: REDIS # Type of the store. REDIS, REDIS_CLUSTER are available options config: # Store specific configuration. See host: localhost port: 6379 # Subscriptions indicate which feature sets needs to be retrieved and used to populate this store - subscriptions: - # Wildcards match all options. No filtering is done. - - name: "*" - project: "*" - name: online_cluster type: REDIS_CLUSTER config: # Store specific configuration. # Connection string specifies the host:port of Redis instances in the redis cluster. connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" read_from: MASTER - subscriptions: - - name: "*" - project: "*" - version: "*" - + # Redis operation timeout in ISO-8601 format + timeout: PT0.5S + - name: bigtable + type: BIGTABLE + config: + project_id: + instance_id: + app_profile_id: + - name: cassandra + type: CASSANDRA + config: + connection_string: localhost:9042 + data_center: datacenter1 + keyspace: feast tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls # which can be useful to debug performance issues and perform benchmarking diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 79d4773..d49ac41 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -51,6 +51,16 @@ public class BaseAuthIT { static final String REDIS = "redis_1"; static final int REDIS_PORT = 6379; + static final String BIGTABLE = "bigtable_1"; + static final int BIGTABLE_PORT = 8086; + + static final String CASSANDRA = "cassandra_1"; + static final int CASSANDRA_PORT = 9042; + static final String CASSANDRA_DATACENTER = "datacenter1"; + static final String CASSANDRA_KEYSPACE = "feast"; + static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference"; + static final String CASSANDRA_ENTITY_KEY = "key"; + static final int FEAST_CORE_PORT = 6565; @DynamicPropertySource @@ -69,8 +79,39 @@ static void properties(DynamicPropertyRegistry registry) { } }); registry.add("feast.stores[0].config.port", () -> REDIS_PORT); - registry.add("feast.stores[0].subscriptions[0].name", () -> "*"); - registry.add("feast.stores[0].subscriptions[0].project", () -> "*"); + + registry.add("feast.stores[1].name", () -> "bigtable"); + registry.add("feast.stores[1].type", () -> "BIGTABLE"); + registry.add("feast.stores[1].config.project_id", () -> "test-project"); + registry.add("feast.stores[1].config.instance_id", () -> "test-instance"); + + registry.add("feast.stores[2].name", () -> "cassandra"); + registry.add("feast.stores[2].type", () -> "CASSANDRA"); + registry.add( + "feast.stores[2].config.host", + () -> { + try { + return InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + return ""; + } + }); + + registry.add( + "feast.stores[2].config.connection_string", + () -> { + String hostAddress = ""; + try { + hostAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + return String.format("%s:%s", hostAddress, CASSANDRA_PORT); + }); + registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); + registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE); registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java b/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java new file mode 100644 index 0000000..21bddb5 --- /dev/null +++ b/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java @@ -0,0 +1,873 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.stub.BigtableTableAdminStubSettings; +import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.protobuf.ByteString; +import feast.common.it.DataGenerator; +import feast.common.models.FeatureV2; +import feast.proto.core.EntityProto; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingServiceGrpc; +import feast.proto.types.ValueProto; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@ActiveProfiles("it") +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "feast.core-cache-refresh-interval=1", + "feast.active_store=bigtable", + "spring.main.allow-bean-definition-overriding=true" + }) +@Testcontainers +public class ServingServiceBigTableIT extends BaseAuthIT { + + static final Map options = new HashMap<>(); + static CoreSimpleAPIClient coreClient; + static ServingServiceGrpc.ServingServiceBlockingStub servingStub; + + static BigtableDataClient client; + static final int FEAST_SERVING_PORT = 6569; + + static final String PROJECT_ID = "test-project"; + static final String INSTANCE_ID = "test-instance"; + static final String APP_PROFILE_ID = "default"; + static ManagedChannel channel; + + static final FeatureReferenceV2 feature1Reference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + static final FeatureReferenceV2 feature2Reference = + DataGenerator.createFeatureReference("rides", "trip_distance"); + static final FeatureReferenceV2 feature3Reference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + static final FeatureReferenceV2 feature4Reference = + DataGenerator.createFeatureReference("rides", "trip_wrong_type"); + + @ClassRule @Container + public static DockerComposeContainer environment = + new DockerComposeContainer( + new File("src/test/resources/docker-compose/docker-compose-bigtable-it.yml")) + .withExposedService( + CORE, + FEAST_CORE_PORT, + Wait.forLogMessage(".*gRPC Server started.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) + .withExposedService(BIGTABLE, BIGTABLE_PORT); + + @DynamicPropertySource + static void initialize(DynamicPropertyRegistry registry) { + registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + } + + @BeforeAll + static void globalSetup() throws IOException { + coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); + servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); + + // Initialize BigTable Client + client = + BigtableDataClient.create( + BigtableDataSettings.newBuilderForEmulator( + environment.getServiceHost("bigtable_1", BIGTABLE_PORT), + environment.getServicePort("bigtable_1", BIGTABLE_PORT)) + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .build()); + + String endpoint = + environment.getServiceHost("bigtable_1", BIGTABLE_PORT) + + ":" + + environment.getServicePort("bigtable_1", BIGTABLE_PORT); + channel = ManagedChannelBuilder.forTarget(endpoint).usePlaintext().build(); + + /** Feast resource creation Workflow */ + String projectName = "default"; + // Apply Entity (driver_id) + String driverEntityName = "driver_id"; + String driverEntityDescription = "My driver id"; + ValueProto.ValueType.Enum driverEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 driverEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(driverEntityName) + .setDescription(driverEntityDescription) + .setValueType(driverEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, driverEntitySpec); + + // Apply Entity (this_is_a_long_long_long_long_long_long_entity_id) + String superLongEntityName = "this_is_a_long_long_long_long_long_long_entity_id"; + String superLongEntityDescription = "My super long entity id"; + ValueProto.ValueType.Enum superLongEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 superLongEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(superLongEntityName) + .setDescription(superLongEntityDescription) + .setValueType(superLongEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, superLongEntitySpec); + + // Apply Entity (merchant_id) + String merchantEntityName = "merchant_id"; + String merchantEntityDescription = "My merchant id"; + ValueProto.ValueType.Enum merchantEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 merchantEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(merchantEntityName) + .setDescription(merchantEntityDescription) + .setValueType(merchantEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, merchantEntitySpec); + + // Apply FeatureTable (rides) + String ridesFeatureTableName = "rides"; + ImmutableList ridesEntities = ImmutableList.of(driverEntityName); + ImmutableMap ridesFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT64, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE, + "trip_empty", + ValueProto.ValueType.Enum.DOUBLE, + "trip_wrong_type", + ValueProto.ValueType.Enum.STRING); + TestUtils.applyFeatureTable( + coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); + + // Apply FeatureTable (superLong) + String superLongFeatureTableName = "superlong"; + ImmutableList superLongEntities = ImmutableList.of(superLongEntityName); + ImmutableMap superLongFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT64, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE, + "trip_empty", + ValueProto.ValueType.Enum.DOUBLE, + "trip_wrong_type", + ValueProto.ValueType.Enum.STRING); + TestUtils.applyFeatureTable( + coreClient, + projectName, + superLongFeatureTableName, + superLongEntities, + superLongFeatures, + 7200); + + // Apply FeatureTable (rides_merchant) + String rideMerchantFeatureTableName = "rides_merchant"; + ImmutableList ridesMerchantEntities = + ImmutableList.of(driverEntityName, merchantEntityName); + TestUtils.applyFeatureTable( + coreClient, + projectName, + rideMerchantFeatureTableName, + ridesMerchantEntities, + ridesFeatures, + 7200); + + // BigTable Table names + String superLongBtTableName = String.format("%s__%s", projectName, superLongEntityName); + String hashSuffix = + Hashing.murmur3_32().hashBytes(superLongBtTableName.substring(42).getBytes()).toString(); + superLongBtTableName = + superLongBtTableName + .substring(0, Math.min(superLongBtTableName.length(), 42)) + .concat(hashSuffix); + String btTableName = String.format("%s__%s", projectName, driverEntityName); + String compoundBtTableName = + String.format( + "%s__%s", + projectName, ridesMerchantEntities.stream().collect(Collectors.joining("__"))); + String featureTableName = "rides"; + String metadataColumnFamily = "metadata"; + ImmutableList columnFamilies = ImmutableList.of(featureTableName, metadataColumnFamily); + ImmutableList compoundColumnFamilies = + ImmutableList.of(rideMerchantFeatureTableName, metadataColumnFamily); + + /** Single Entity Ingestion Workflow */ + Schema ftSchema = + SchemaBuilder.record("DriverData") + .namespace(featureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] schemaReference = + Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); + + GenericRecord record = + new GenericRecordBuilder(ftSchema) + .set("trip_cost", 5L) + .set("trip_distance", 3.5) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] entityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] entityFeatureValue = createEntityValue(ftSchema, schemaReference, record); + byte[] schemaKey = createSchemaKey(schemaReference); + ingestData( + featureTableName, btTableName, entityFeatureKey, entityFeatureValue, schemaKey, ftSchema); + + /** SuperLong Entity Ingestion Workflow */ + Schema superLongFtSchema = + SchemaBuilder.record("SuperLongData") + .namespace(superLongFeatureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] superLongSchemaReference = + Hashing.murmur3_32().hashBytes(superLongFtSchema.toString().getBytes()).asBytes(); + + GenericRecord superLongRecord = + new GenericRecordBuilder(superLongFtSchema) + .set("trip_cost", 5L) + .set("trip_distance", 3.5) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] superLongEntityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] superLongEntityFeatureValue = + createEntityValue(superLongFtSchema, superLongSchemaReference, superLongRecord); + byte[] superLongSchemaKey = createSchemaKey(superLongSchemaReference); + ingestData( + superLongFeatureTableName, + superLongBtTableName, + superLongEntityFeatureKey, + superLongEntityFeatureValue, + superLongSchemaKey, + superLongFtSchema); + + /** Compound Entity Ingestion Workflow */ + Schema compoundFtSchema = + SchemaBuilder.record("DriverMerchantData") + .namespace(rideMerchantFeatureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] compoundSchemaReference = + Hashing.murmur3_32().hashBytes(compoundFtSchema.toString().getBytes()).asBytes(); + + // Entity-Feature Row + GenericRecord compoundEntityRecord = + new GenericRecordBuilder(compoundFtSchema) + .set("trip_cost", 10L) + .set("trip_distance", 5.5) + .set("trip_empty", null) + .set("trip_wrong_type", "wrong_type") + .build(); + + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + byte[] compoundEntityFeatureKey = + ridesMerchantEntities.stream() + .map(entity -> DataGenerator.valueToString(entityRow.getFieldsMap().get(entity))) + .collect(Collectors.joining("#")) + .getBytes(); + byte[] compoundEntityFeatureValue = + createEntityValue(compoundFtSchema, compoundSchemaReference, compoundEntityRecord); + byte[] compoundSchemaKey = createSchemaKey(compoundSchemaReference); + ingestData( + rideMerchantFeatureTableName, + compoundBtTableName, + compoundEntityFeatureKey, + compoundEntityFeatureValue, + compoundSchemaKey, + compoundFtSchema); + + // set up options for call credentials + options.put("oauth_url", TOKEN_URL); + options.put(CLIENT_ID, CLIENT_ID); + options.put(CLIENT_SECRET, CLIENT_SECRET); + options.put("jwkEndpointURI", JWK_URI); + options.put("audience", AUDIENCE); + options.put("grant_type", GRANT_TYPE); + } + + @AfterAll + static void tearDown() { + ((ManagedChannel) servingStub.getChannel()).shutdown(); + channel.shutdown(); + } + + private static void createTable( + TransportChannelProvider channelProvider, + CredentialsProvider credentialsProvider, + String tableName, + List columnFamilies) + throws IOException { + EnhancedBigtableTableAdminStub stub = + EnhancedBigtableTableAdminStub.createEnhanced( + BigtableTableAdminStubSettings.newBuilder() + .setTransportChannelProvider(channelProvider) + .setCredentialsProvider(credentialsProvider) + .build()); + + try (BigtableTableAdminClient client = + BigtableTableAdminClient.create(PROJECT_ID, INSTANCE_ID, stub)) { + CreateTableRequest createTableRequest = CreateTableRequest.of(tableName); + for (String columnFamily : columnFamilies) { + createTableRequest.addFamily(columnFamily); + } + if (!client.exists(tableName)) { + client.createTable(createTableRequest); + } + } + } + + private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { + String schemaKeyPrefix = "schema#"; + + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaKeyPrefix.getBytes()); + concatOutputStream.write(schemaReference); + byte[] schemaKey = concatOutputStream.toByteArray(); + + return schemaKey; + } + + private static byte[] createEntityValue( + Schema schema, byte[] schemaReference, GenericRecord record) throws IOException { + // Entity-Feature Row + byte[] avroSerializedFeatures = recordToAvro(record, schema); + + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaReference); + concatOutputStream.write("".getBytes()); + concatOutputStream.write(avroSerializedFeatures); + byte[] entityFeatureValue = concatOutputStream.toByteArray(); + + return entityFeatureValue; + } + + private static byte[] schemaReference(Schema schema) { + return Hashing.murmur3_32().hashBytes(schema.toString().getBytes()).asBytes(); + } + + private static void ingestData( + String featureTableName, + String btTableName, + byte[] btEntityFeatureKey, + byte[] btEntityFeatureValue, + byte[] btSchemaKey, + Schema btSchema) + throws IOException { + String emptyQualifier = ""; + String avroQualifier = "avro"; + String metadataColumnFamily = "metadata"; + + TransportChannelProvider channelProvider = + FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); + NoCredentialsProvider credentialsProvider = NoCredentialsProvider.create(); + createTable( + channelProvider, + credentialsProvider, + btTableName, + ImmutableList.of(featureTableName, metadataColumnFamily)); + + // Update Compound Entity-Feature Row + client.mutateRow( + RowMutation.create(btTableName, ByteString.copyFrom(btEntityFeatureKey)) + .setCell( + featureTableName, + ByteString.copyFrom(emptyQualifier.getBytes()), + ByteString.copyFrom(btEntityFeatureValue))); + + // Update Schema Row + client.mutateRow( + RowMutation.create(btTableName, ByteString.copyFrom(btSchemaKey)) + .setCell( + metadataColumnFamily, + ByteString.copyFrom(avroQualifier.getBytes()), + ByteString.copyFrom(btSchema.toString().getBytes()))); + } + + private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); + writer.write(datum, encoder); + encoder.flush(); + + return output.toByteArray(); + } + + @Test + public void shouldRegisterSingleEntityAndGetOnlineFeatures() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); + ImmutableList entityRows = ImmutableList.of(entityRow1); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { + String projectName = "default"; + String driverEntityName = "driver_id"; + String merchantEntityName = "merchant_id"; + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + ImmutableList entityRows = ImmutableList.of(entityRow); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + driverEntityName, + driverEntityValue, + merchantEntityName, + merchantEntityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + driverEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + merchantEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldReturnCorrectRowCount() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, entityValue1, 100); + GetOnlineFeaturesRequestV2.EntityRow entityRow2 = + DataGenerator.createEntityRow(entityName, entityValue2, 100); + ImmutableList entityRows = + ImmutableList.of(entityRow1, entityRow2); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + FeatureReferenceV2 emptyFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue1, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + + ImmutableMap expectedValueMap2 = + ImmutableMap.of( + entityName, + entityValue2, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap2 = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap2) + .putAllStatuses(expectedStatusMap2) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues, expectedFieldValues2); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldSupportAllFeastTypes() throws IOException { + EntityProto.EntitySpecV2 entitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName("entity") + .setDescription("") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build(); + TestUtils.applyEntity(coreClient, "default", entitySpec); + + ImmutableMap allTypesFeatures = + new ImmutableMap.Builder() + .put("f_int64", ValueProto.ValueType.Enum.INT64) + .put("f_int32", ValueProto.ValueType.Enum.INT32) + .put("f_float", ValueProto.ValueType.Enum.FLOAT) + .put("f_double", ValueProto.ValueType.Enum.DOUBLE) + .put("f_string", ValueProto.ValueType.Enum.STRING) + .put("f_bytes", ValueProto.ValueType.Enum.BYTES) + .put("f_bool", ValueProto.ValueType.Enum.BOOL) + .put("f_int64_list", ValueProto.ValueType.Enum.INT64_LIST) + .put("f_int32_list", ValueProto.ValueType.Enum.INT32_LIST) + .put("f_float_list", ValueProto.ValueType.Enum.FLOAT_LIST) + .put("f_double_list", ValueProto.ValueType.Enum.DOUBLE_LIST) + .put("f_string_list", ValueProto.ValueType.Enum.STRING_LIST) + .put("f_bytes_list", ValueProto.ValueType.Enum.BYTES_LIST) + .put("f_bool_list", ValueProto.ValueType.Enum.BOOL_LIST) + .build(); + + TestUtils.applyFeatureTable( + coreClient, "default", "all_types", ImmutableList.of("entity"), allTypesFeatures, 7200); + + Schema schema = + SchemaBuilder.record("AllTypesRecord") + .namespace("") + .fields() + .requiredLong("f_int64") + .requiredInt("f_int32") + .requiredFloat("f_float") + .requiredDouble("f_double") + .requiredString("f_string") + .requiredBytes("f_bytes") + .requiredBoolean("f_bool") + .name("f_int64_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().longType())) + .noDefault() + .name("f_int32_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().intType())) + .noDefault() + .name("f_float_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().floatType())) + .noDefault() + .name("f_double_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().doubleType())) + .noDefault() + .name("f_string_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())) + .noDefault() + .name("f_bytes_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().bytesType())) + .noDefault() + .name("f_bool_list") + .type(SchemaBuilder.array().items(SchemaBuilder.builder().booleanType())) + .noDefault() + .endRecord(); + + GenericData.Record record = + new GenericRecordBuilder(schema) + .set("f_int64", 10L) + .set("f_int32", 10) + .set("f_float", 10.0) + .set("f_double", 10.0D) + .set("f_string", "test") + .set("f_bytes", ByteBuffer.wrap("test".getBytes())) + .set("f_bool", true) + .set("f_int64_list", ImmutableList.of(10L)) + .set("f_int32_list", ImmutableList.of(10)) + .set("f_float_list", ImmutableList.of(10.0)) + .set("f_double_list", ImmutableList.of(10.0D)) + .set("f_string_list", ImmutableList.of("test")) + .set("f_bytes_list", ImmutableList.of(ByteBuffer.wrap("test".getBytes()))) + .set("f_bool_list", ImmutableList.of(true)) + .build(); + + ValueProto.Value entity = DataGenerator.createStrValue("key"); + + ingestData( + "all_types", + "default__entity", + entity.getStringVal().getBytes(), + createEntityValue(schema, schemaReference(schema), record), + createSchemaKey(schemaReference(schema)), + schema); + + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest( + "default", + allTypesFeatures.keySet().stream() + .map( + f -> + FeatureReferenceV2.newBuilder() + .setFeatureTable("all_types") + .setName(f) + .build()) + .collect(Collectors.toList()), + ImmutableList.of(DataGenerator.createEntityRow("entity", entity, 100))); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + assert featureResponse.getFieldValues(0).getStatusesMap().values().stream() + .allMatch(status -> status.equals(GetOnlineFeaturesResponse.FieldStatus.PRESENT)); + } + + @Test + public void shouldRegisterSuperLongEntityAndGetOnlineFeatures() { + // getOnlineFeatures Information + String projectName = "default"; + String entityName = "this_is_a_long_long_long_long_long_long_entity_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); + ImmutableList entityRows = ImmutableList.of(entityRow1); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("superlong", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("superlong", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @TestConfiguration + public static class TestConfig { + @Bean + public BigtableDataClient bigtableClient() throws IOException { + return BigtableDataClient.create( + BigtableDataSettings.newBuilderForEmulator( + environment.getServiceHost("bigtable_1", BIGTABLE_PORT), + environment.getServicePort("bigtable_1", BIGTABLE_PORT)) + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setAppProfileId(APP_PROFILE_ID) + .build()); + } + } +} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceIT.java b/serving/src/test/java/feast/serving/it/ServingServiceIT.java deleted file mode 100644 index f08ff88..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceIT.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.*; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.storage.RedisProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.*; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - }) -@Testcontainers -public class ServingServiceIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static final String timestampPrefix = "_ts"; - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - static RedisCommands syncCommands; - - static final int FEAST_SERVING_PORT = 6568; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(REDIS, REDIS_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - RedisClient redisClient = - RedisClient.create( - new RedisURI( - environment.getServiceHost("redis_1", REDIS_PORT), - environment.getServicePort("redis_1", REDIS_PORT), - java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - syncCommands = connection.sync(); - - String projectName = "default"; - // Apply Entity - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - String description = "My driver id"; - ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(entityName) - .setDescription(description) - .setValueType(entityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, entitySpec); - - // Apply FeatureTable - String featureTableName = "rides"; - ImmutableList entities = ImmutableList.of(entityName); - - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - ServingAPIProto.FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - // Event Timestamp - String eventTimestampKey = timestampPrefix + ":" + featureTableName; - Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); - - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Serialize Redis Key with Entity i.e - RedisProto.RedisKeyV2 redisKey = - RedisProto.RedisKeyV2.newBuilder() - .setProject(projectName) - .addEntityNames(entityName) - .addEntityValues(entityValue) - .build(); - - ImmutableMap featureReferenceValueMap = - ImmutableMap.of( - feature1Reference, - DataGenerator.createInt64Value(42), - feature2Reference, - DataGenerator.createDoubleValue(42.2), - feature3Reference, - DataGenerator.createEmptyValue(), - feature4Reference, - DataGenerator.createDoubleValue(42.2)); - - // Insert timestamp into Redis and isTimestampMap only once - syncCommands.hset( - redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); - featureReferenceValueMap.forEach( - (featureReference, featureValue) -> { - // Murmur hash Redis Feature Field i.e murmur() - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - byte[] featureReferenceBytes = - Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - // Insert features into Redis - syncCommands.hset( - redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); - }); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldRegisterAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ImmutableList featureReferences = - ImmutableList.of(feature1Reference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(feature1Reference), - DataGenerator.createInt64Value(42)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(feature1Reference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldRegisterAndGetOnlineFeaturesWithNotFound() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - ServingAPIProto.FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(42), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldGetOnlineFeaturesOutsideMaxAge() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 7400); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForDiffType() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForUpdatedType() { - String projectName = "default"; - String entityName = "driver_id"; - String featureTableName = "rides"; - - ImmutableList entities = ImmutableList.of(entityName); - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.STRING, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Sleep is necessary to ensure caching (every 1s) of updated FeatureTable is done - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - } - - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java deleted file mode 100644 index 8f2440d..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import com.google.common.collect.ImmutableMap; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false" - }) -@Testcontainers -public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { - - CoreSimpleAPIClient coreClient; - FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; - static final Map options = new HashMap<>(); - - static final int FEAST_SERVING_PORT = 6566; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @BeforeEach - public void initState() { - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - EntityProto.EntitySpecV2 entitySpec = - DataGenerator.createEntitySpecV2( - ENTITY_ID, - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); - - expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - Arrays.asList(ENTITY_ID), - new HashMap<>() { - { - put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldAllowUnauthenticatedGetOnlineFeatures() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); - - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAuthenticated() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java deleted file mode 100644 index 64fe44b..0000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import feast.common.it.DataGenerator; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import sh.ory.keto.ApiException; - -@ActiveProfiles("it") -@SpringBootTest( - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true" - }) -@Testcontainers -public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { - - static final Map adminCredentials = new HashMap<>(); - static final Map memberCredentials = new HashMap<>(); - static final String PROJECT_MEMBER_CLIENT_ID = "client_id_1"; - static final String NOT_PROJECT_MEMBER_CLIENT_ID = "client_id_2"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - static String subjectClaim = "sub"; - static CoreSimpleAPIClient coreClient; - static final int FEAST_SERVING_PORT = 6766; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml"), - new File("src/test/resources/docker-compose/docker-compose-it-keto.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - AuthTestUtils.seedKeto(ketoExternalUrl, PROJECT_NAME, PROJECT_MEMBER_CLIENT_ID, CLIENT_ID); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, NOT_PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - // set up options for call credentials - adminCredentials.put("oauth_url", TOKEN_URL); - adminCredentials.put(CLIENT_ID, CLIENT_ID); - adminCredentials.put(CLIENT_SECRET, CLIENT_SECRET); - adminCredentials.put("jwkEndpointURI", JWK_URI); - adminCredentials.put("audience", AUDIENCE); - adminCredentials.put("grant_type", GRANT_TYPE); - - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); - coreClient.simpleApplyEntity( - PROJECT_NAME, - DataGenerator.createEntitySpecV2( - ENTITY_ID, "", ValueProto.ValueType.Enum.STRING, Collections.emptyMap())); - coreClient.simpleApplyFeatureTable( - PROJECT_NAME, - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - ImmutableList.of(ENTITY_ID), - ImmutableMap.of(FEATURE_NAME, ValueProto.ValueType.Enum.STRING), - 0, - Collections.emptyMap())); - } - - @Test - public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAdmin() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfProjectMember() { - Map memberCredsOptions = new HashMap<>(); - memberCredsOptions.putAll(adminCredentials); - memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void cantGetOnlineFeaturesIfNotProjectMember() { - Map notMemberCredsOptions = new HashMap<>(); - notMemberCredsOptions.putAll(adminCredentials); - notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", - PROJECT_NAME, NOT_PROJECT_MEMBER_CLIENT_ID); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index 57932d4..4e48b64 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -18,8 +18,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -32,7 +30,6 @@ import feast.proto.core.CoreServiceProto.ListProjectsResponse; import feast.proto.core.FeatureTableProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto.Store; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; @@ -45,8 +42,6 @@ public class CachedSpecServiceTest { - private Store store; - @Rule public final ExpectedException expectedException = ExpectedException.none(); @Mock CoreSpecService coreService; @@ -63,8 +58,6 @@ public class CachedSpecServiceTest { public void setUp() { initMocks(this); - this.store = Store.newBuilder().build(); - this.setupProject("default"); this.featureTableEntities = ImmutableList.of("entity1"); this.featureTable1Features = @@ -94,8 +87,7 @@ public void setUp() { this.setupFeatureTableAndProject("default"); - when(this.coreService.registerStore(store)).thenReturn(store); - cachedSpecService = new CachedSpecService(this.coreService, this.store); + cachedSpecService = new CachedSpecService(this.coreService); } private void setupProject(String project) { @@ -120,18 +112,6 @@ private void setupFeatureTableAndProject(String project) { .build()); } - @Test - public void shouldRegisterStoreWithCore() { - verify(coreService, times(1)).registerStore(cachedSpecService.getStore()); - } - - @Test - public void shouldPopulateAndReturnStore() { - cachedSpecService.populateCache(); - Store actual = cachedSpecService.getStore(); - assertThat(actual, equalTo(store)); - } - @Test public void shouldPopulateAndReturnDifferentFeatureTables() { // test that CachedSpecService can retrieve fully qualified feature references. diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 539ed39..d3e62b4 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -34,7 +34,9 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.ProtoFeature; import feast.storage.connectors.redis.retriever.OnlineRetriever; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; @@ -51,6 +53,7 @@ public class OnlineServingServiceTest { @Mock CachedSpecService specService; @Mock Tracer tracer; @Mock OnlineRetriever retrieverV2; + private String transformationServiceEndpoint; private OnlineServingServiceV2 onlineServingServiceV2; @@ -60,69 +63,62 @@ public class OnlineServingServiceTest { @Before public void setUp() { initMocks(this); - onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); + CoreFeatureSpecRetriever coreFeatureSpecRetriever = new CoreFeatureSpecRetriever(specService); + OnlineTransformationService onlineTransformationService = + new OnlineTransformationService(transformationServiceEndpoint, coreFeatureSpecRetriever); + onlineServingServiceV2 = + new OnlineServingServiceV2( + retrieverV2, tracer, coreFeatureSpecRetriever, onlineTransformationService); mockedFeatureRows = new ArrayList<>(); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("1")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("1"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("2")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("2"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("3")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("3"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("4")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("4"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_3") - .build()) - .setFeatureValue(createStrValue("5")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_3") + .build(), + Timestamp.newBuilder().setSeconds(100).build(), + createStrValue("5"))); mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("6")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(50).build()) - .build()); + new ProtoFeature( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(), + Timestamp.newBuilder().setSeconds(50).build(), + createStrValue("6"))); featureSpecs = new ArrayList<>(); featureSpecs.add( @@ -163,7 +159,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); @@ -230,7 +226,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); @@ -294,7 +290,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { List> featureRows = List.of(entityKeyList1, entityKeyList2); - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); when(specService.getFeatureTableSpec(any(), any())) .thenReturn( FeatureTableSpec.newBuilder() diff --git a/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml b/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml new file mode 100644 index 0000000..28985ef --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml @@ -0,0 +1,38 @@ +version: '3' + +services: + core: + image: gcr.io/kf-feast/feast-core:develop + volumes: + - ./core/application-it.yml:/etc/feast/application.yml + environment: + DB_HOST: db + restart: on-failure + depends_on: + - db + ports: + - 6565:6565 + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + + bigtable: + image: google/cloud-sdk:latest + environment: + GOOGLE_APPLICATION_CREDENTIALS: /Users/user/.config/gcloud/application_default_credentials.json + command: + - gcloud + - beta + - emulators + - bigtable + - start + - --host-port=0.0.0.0:8086 \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml new file mode 100644 index 0000000..15afad0 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml @@ -0,0 +1,31 @@ +version: '3' + +services: + core: + image: gcr.io/kf-feast/feast-core:develop + volumes: + - ./core/application-it.yml:/etc/feast/application.yml + environment: + DB_HOST: db + restart: on-failure + depends_on: + - db + ports: + - 6565:6565 + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + + cassandra: + image: datastax/cassandra:4.0 + ports: + - "9042:9042" \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml new file mode 100644 index 0000000..33d65f4 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + redis: + image: redis:5-alpine + ports: + - "6379:6379" + materialize: + build: feast10 + links: + - redis + diff --git a/serving/src/test/resources/docker-compose/feast10/Dockerfile b/serving/src/test/resources/docker-compose/feast10/Dockerfile new file mode 100644 index 0000000..bde9f11 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7 + +WORKDIR /usr/src/ + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "./materialize.py" ] diff --git a/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet b/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet new file mode 100644 index 0000000..df8cbba Binary files /dev/null and b/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet differ diff --git a/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml new file mode 100644 index 0000000..87e3310 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml @@ -0,0 +1,9 @@ +project: feast_project +provider: local +online_store: + type: redis + connection_string: "redis:6379" +offline_store: {} +flags: + alpha_features: true + on_demand_transforms: true diff --git a/serving/src/test/resources/docker-compose/feast10/materialize.py b/serving/src/test/resources/docker-compose/feast10/materialize.py new file mode 100644 index 0000000..6338c16 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/materialize.py @@ -0,0 +1,45 @@ +# This is an example feature definition file + +from google.protobuf.duration_pb2 import Duration + +from datetime import datetime +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, FeatureStore + +print("Running materialize.py") + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +file_path = "driver_stats.parquet" +driver_hourly_stats = FileSource( + path=file_path, + event_timestamp_column="event_timestamp", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=Duration(seconds=86400 * 365), + features=[ + Feature(name="conv_rate", dtype=ValueType.DOUBLE), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), + ], + online=True, + batch_source=driver_hourly_stats, + tags={}, +) + +fs = FeatureStore(".") +fs.apply([driver_hourly_stats_view, driver]) + +now = datetime.now() +fs.materialize_incremental(now) diff --git a/serving/src/test/resources/docker-compose/feast10/registry.db b/serving/src/test/resources/docker-compose/feast10/registry.db new file mode 100644 index 0000000..774b493 Binary files /dev/null and b/serving/src/test/resources/docker-compose/feast10/registry.db differ diff --git a/serving/src/test/resources/docker-compose/feast10/requirements.txt b/serving/src/test/resources/docker-compose/feast10/requirements.txt new file mode 100644 index 0000000..cb579a2 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/requirements.txt @@ -0,0 +1 @@ +feast[redis]>=0.13,<1 diff --git a/storage/api/pom.xml b/storage/api/pom.xml index cc2f84e..4e7ad39 100644 --- a/storage/api/pom.xml +++ b/storage/api/pom.xml @@ -61,6 +61,12 @@ 3.9 + + org.apache.avro + avro + 1.10.2 + + junit junit diff --git a/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java b/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java new file mode 100644 index 0000000..96f19cc --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.retriever; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; +import java.nio.ByteBuffer; +import java.util.stream.Collectors; +import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; + +public class AvroFeature implements Feature { + private final ServingAPIProto.FeatureReferenceV2 featureReference; + + private final Timestamp eventTimestamp; + + private final Object featureValue; + + public AvroFeature( + ServingAPIProto.FeatureReferenceV2 featureReference, + Timestamp eventTimestamp, + Object featureValue) { + this.featureReference = featureReference; + this.eventTimestamp = eventTimestamp; + this.featureValue = featureValue; + } + + /** + * Casts feature value of Object type based on Feast valueType. Empty object i.e new Object() is + * interpreted as VAL_NOT_SET Feast valueType. + * + * @param valueType Feast valueType of feature as specified in FeatureSpec + * @return ValueProto.Value representation of feature + */ + @Override + public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) { + ValueProto.Value finalValue; + + try { + switch (valueType) { + case STRING: + finalValue = + ValueProto.Value.newBuilder().setStringVal(((Utf8) featureValue).toString()).build(); + break; + case INT32: + finalValue = ValueProto.Value.newBuilder().setInt32Val((Integer) featureValue).build(); + break; + case INT64: + finalValue = ValueProto.Value.newBuilder().setInt64Val((Long) featureValue).build(); + break; + case DOUBLE: + finalValue = ValueProto.Value.newBuilder().setDoubleVal((Double) featureValue).build(); + break; + case FLOAT: + finalValue = ValueProto.Value.newBuilder().setFloatVal((Float) featureValue).build(); + break; + case BYTES: + finalValue = + ValueProto.Value.newBuilder() + .setBytesVal(ByteString.copyFrom(((ByteBuffer) featureValue).array())) + .build(); + break; + case BOOL: + finalValue = ValueProto.Value.newBuilder().setBoolVal((Boolean) featureValue).build(); + break; + case STRING_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setStringListVal( + ValueProto.StringList.newBuilder() + .addAllVal( + ((GenericData.Array) featureValue) + .stream().map(Utf8::toString).collect(Collectors.toList())) + .build()) + .build(); + break; + case INT64_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setInt64ListVal( + ValueProto.Int64List.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case INT32_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setInt32ListVal( + ValueProto.Int32List.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case FLOAT_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setFloatListVal( + ValueProto.FloatList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case DOUBLE_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setDoubleListVal( + ValueProto.DoubleList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case BOOL_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setBoolListVal( + ValueProto.BoolList.newBuilder() + .addAllVal(((GenericData.Array) featureValue)) + .build()) + .build(); + break; + case BYTES_LIST: + finalValue = + ValueProto.Value.newBuilder() + .setBytesListVal( + ValueProto.BytesList.newBuilder() + .addAllVal( + ((GenericData.Array) featureValue) + .stream() + .map(byteBuffer -> ByteString.copyFrom(byteBuffer.array())) + .collect(Collectors.toList())) + .build()) + .build(); + break; + default: + throw new RuntimeException("FeatureType is not supported"); + } + } catch (ClassCastException e) { + // Feature type has changed + finalValue = ValueProto.Value.newBuilder().build(); + } + + return finalValue; + } + + @Override + public ServingAPIProto.FeatureReferenceV2 getFeatureReference() { + return this.featureReference; + } + + @Override + public Timestamp getEventTimestamp() { + return this.eventTimestamp; + } +} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java b/storage/api/src/main/java/feast/storage/api/retriever/Feature.java index c6cee08..92ae1f3 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/Feature.java @@ -16,33 +16,37 @@ */ package feast.storage.api.retriever; -import com.google.auto.value.AutoValue; import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; - -@AutoValue -public abstract class Feature { - - public abstract FeatureReferenceV2 getFeatureReference(); - - public abstract Value getFeatureValue(); - - public abstract Timestamp getEventTimestamp(); - - public static Builder builder() { - return new AutoValue_Feature.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setFeatureReference(FeatureReferenceV2 featureReference); - - public abstract Builder setFeatureValue(Value featureValue); - - public abstract Builder setEventTimestamp(Timestamp eventTimestamp); - - public abstract Feature build(); - } +import java.util.HashMap; + +public interface Feature { + + HashMap TYPE_TO_VAL_CASE = + new HashMap() { + { + put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); + put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); + put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); + put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); + put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); + put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); + put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); + put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); + put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); + put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); + put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); + put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); + put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); + put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); + } + }; + + Value getFeatureValue(ValueProto.ValueType.Enum valueType); + + FeatureReferenceV2 getFeatureReference(); + + Timestamp getEventTimestamp(); } diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java index 9be66a7..a49ab3f 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java @@ -33,11 +33,13 @@ public interface OnlineRetrieverV2 { * @param project name of project to request features from. * @param entityRows list of entity rows to request features for. * @param featureReferences specifies the FeatureTable to retrieve data from + * @param entityNames name of entities * @return list of {@link Feature}s corresponding to data retrieved for each entity row from * FeatureTable specified in FeatureTable request. */ List> getOnlineFeatures( String project, List entityRows, - List featureReferences); + List featureReferences, + List entityNames); } diff --git a/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java b/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java new file mode 100644 index 0000000..09f6b75 --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.retriever; + +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; + +public class ProtoFeature implements Feature { + private final ServingAPIProto.FeatureReferenceV2 featureReference; + + private final Timestamp eventTimestamp; + + private final ValueProto.Value featureValue; + + public ProtoFeature( + ServingAPIProto.FeatureReferenceV2 featureReference, + Timestamp eventTimestamp, + ValueProto.Value featureValue) { + this.featureReference = featureReference; + this.eventTimestamp = eventTimestamp; + this.featureValue = featureValue; + } + + /** + * Returns Feast valueType if type matches, otherwise null. + * + * @param valueType Feast valueType of feature as specified in FeatureSpec + * @return ValueProto.Value representation of feature + */ + @Override + public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) { + if (TYPE_TO_VAL_CASE.get(valueType) != this.featureValue.getValCase()) { + return null; + } + + return this.featureValue; + } + + @Override + public ServingAPIProto.FeatureReferenceV2 getFeatureReference() { + return this.featureReference; + } + + @Override + public Timestamp getEventTimestamp() { + return this.eventTimestamp; + } +} diff --git a/storage/connectors/bigtable/pom.xml b/storage/connectors/bigtable/pom.xml new file mode 100644 index 0000000..81cd450 --- /dev/null +++ b/storage/connectors/bigtable/pom.xml @@ -0,0 +1,45 @@ + + + + dev.feast + feast-storage-connectors + ${revision} + + + 4.0.0 + feast-storage-connector-bigtable + + + 11 + 11 + + + + + com.google.cloud + google-cloud-bigtable + 1.21.2 + + + + + org.apache.avro + avro + 1.10.2 + + + + dev.feast + feast-storage-connector-sstable + ${project.version} + + + + com.google.guava + guava + + + + \ No newline at end of file diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java new file mode 100644 index 0000000..6e67782 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -0,0 +1,211 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.storage.api.retriever.AvroFeature; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; +import java.io.IOException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; + +public class BigTableOnlineRetriever implements SSTableOnlineRetriever { + + private BigtableDataClient client; + private BigTableSchemaRegistry schemaRegistry; + + public BigTableOnlineRetriever(BigtableDataClient client) { + this.client = client; + this.schemaRegistry = new BigTableSchemaRegistry(client); + } + + /** + * Generate BigTable key in the form of entity values joined by #. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return BigTable key for retrieval + */ + @Override + public ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) { + return ByteString.copyFrom( + entityNames.stream() + .map(entity -> entityRow.getFieldsMap().get(entity)) + .map(this::valueToString) + .collect(Collectors.joining("#")) + .getBytes()); + } + + /** + * Converts rowCell feature value into @NativeFeature type. + * + * @param tableName Name of BigTable table + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + @Override + public List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences) { + + BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); + + return rowKeys.stream() + .map( + rowKey -> { + if (!rows.containsKey(rowKey)) { + return Collections.emptyList(); + } else { + Row row = rows.get(rowKey); + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .map(cf -> row.getCells(cf, "")) + .filter(ls -> !ls.isEmpty()) + .flatMap( + rowCells -> { + RowCell rowCell = rowCells.get(0); // Latest cell + String family = rowCell.getFamily(); + ByteString value = rowCell.getValue(); + + List features; + List localFeatureReferences = + featureReferences.stream() + .filter( + featureReference -> + featureReference.getFeatureTable().equals(family)) + .collect(Collectors.toList()); + + try { + features = + decodeFeatures( + tableName, + value, + localFeatureReferences, + reusedDecoder, + rowCell.getTimestamp()); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from BigTable"); + } + + return features.stream(); + }) + .collect(Collectors.toList()); + } + }) + .collect(Collectors.toList()); + } + + /** + * Retrieve rows for each row entity key by generating BigTable rowQuery with filters based on + * column families. + * + * @param tableName Name of BigTable table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + Query rowQuery = Query.create(tableName); + Filters.InterleaveFilter familyFilter = Filters.FILTERS.interleave(); + columnFamilies.forEach(cf -> familyFilter.filter(Filters.FILTERS.family().exactMatch(cf))); + + for (ByteString rowKey : rowKeys) { + rowQuery.rowKey(rowKey); + } + + return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false) + .collect(Collectors.toMap(Row::getKey, Function.identity())); + } + + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is + * returned when null is retrieved from BigTable RowCell. + * + * @param tableName Name of BigTable table + * @param value Value of BigTable cell where first 4 bytes represent the schema reference and + * remaining bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in BigTable RowCell + * @throws IOException + */ + private List decodeFeatures( + String tableName, + ByteString value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + ByteString schemaReferenceBytes = value.substring(0, 4); + byte[] featureValueBytes = value.substring(4).toByteArray(); + + BigTableSchemaRegistry.SchemaReference schemaReference = + new BigTableSchemaRegistry.SchemaReference(tableName, schemaReferenceBytes); + + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + + reusedDecoder = DecoderFactory.get().binaryDecoder(featureValueBytes, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java new file mode 100644 index 0000000..64989f0 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableSchemaRegistry.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Iterables; +import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutionException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; + +public class BigTableSchemaRegistry { + private final BigtableDataClient client; + private final LoadingCache> cache; + + private static String COLUMN_FAMILY = "metadata"; + private static String QUALIFIER = "avro"; + private static String KEY_PREFIX = "schema#"; + + public static class SchemaReference { + private final String tableName; + private final ByteString schemaHash; + + public SchemaReference(String tableName, ByteString schemaHash) { + this.tableName = tableName; + this.schemaHash = schemaHash; + } + + public String getTableName() { + return tableName; + } + + public ByteString getSchemaHash() { + return schemaHash; + } + + @Override + public int hashCode() { + int result = tableName.hashCode(); + result = 31 * result + schemaHash.hashCode(); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SchemaReference that = (SchemaReference) o; + + if (!tableName.equals(that.tableName)) return false; + return schemaHash.equals(that.schemaHash); + } + } + + public BigTableSchemaRegistry(BigtableDataClient client) { + this.client = client; + + CacheLoader> schemaCacheLoader = + CacheLoader.from(this::loadReader); + + cache = CacheBuilder.newBuilder().build(schemaCacheLoader); + } + + public GenericDatumReader getReader(SchemaReference reference) { + GenericDatumReader reader; + try { + reader = this.cache.get(reference); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { + throw new RuntimeException(String.format("Unable to find Schema"), e); + } + return reader; + } + + private GenericDatumReader loadReader(SchemaReference reference) { + Row row = + client.readRow( + reference.getTableName(), + ByteString.copyFrom(KEY_PREFIX.getBytes()).concat(reference.getSchemaHash()), + Filters.FILTERS.family().exactMatch(COLUMN_FAMILY)); + RowCell last = Iterables.getLast(row.getCells(COLUMN_FAMILY, QUALIFIER)); + + Schema schema = new Schema.Parser().parse(last.getValue().toStringUtf8()); + return new GenericDatumReader<>(schema); + } +} diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java new file mode 100644 index 0000000..11a0445 --- /dev/null +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableStoreConfig.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigtable.retriever; + +public class BigTableStoreConfig { + private final String projectId; + private final String instanceId; + private final String appProfileId; + + public BigTableStoreConfig(String projectId, String instanceId, String appProfileId) { + this.projectId = projectId; + this.instanceId = instanceId; + this.appProfileId = appProfileId; + } + + public String getProjectId() { + return this.projectId; + } + + public String getInstanceId() { + return this.instanceId; + } + + public String getAppProfileId() { + return this.appProfileId; + } +} diff --git a/storage/connectors/cassandra/pom.xml b/storage/connectors/cassandra/pom.xml new file mode 100644 index 0000000..32e4d73 --- /dev/null +++ b/storage/connectors/cassandra/pom.xml @@ -0,0 +1,45 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-cassandra + + + 11 + 11 + + + + + org.apache.avro + avro + 1.10.2 + + + + dev.feast + feast-storage-connector-sstable + ${project.version} + + + + com.datastax.oss + java-driver-core + 4.11.0 + + + + com.datastax.oss + java-driver-query-builder + 4.11.0 + + + + \ No newline at end of file diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java new file mode 100644 index 0000000..9b9de7b --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -0,0 +1,271 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.AsyncPagingIterable; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.google.protobuf.Timestamp; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.storage.api.retriever.AvroFeature; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DecoderFactory; + +public class CassandraOnlineRetriever implements SSTableOnlineRetriever { + + private final CqlSession session; + private final CassandraSchemaRegistry schemaRegistry; + + private static final String ENTITY_KEY = "key"; + private static final String SCHEMA_REF_SUFFIX = "__schema_ref"; + private static final String EVENT_TIMESTAMP_SUFFIX = "__event_timestamp"; + private static final int MAX_TABLE_NAME_LENGTH = 48; + + public CassandraOnlineRetriever(CqlSession session) { + this.session = session; + this.schemaRegistry = new CassandraSchemaRegistry(session); + } + + /** + * Generate Cassandra key in the form of entity values joined by #. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return Cassandra key for retrieval + */ + @Override + public ByteBuffer convertEntityValueToKey(EntityRow entityRow, List entityNames) { + return ByteBuffer.wrap( + entityNames.stream() + .map(entity -> entityRow.getFieldsMap().get(entity)) + .map(this::valueToString) + .collect(Collectors.joining("#")) + .getBytes()); + } + + /** + * Generate Cassandra table name, with limit of 48 characters. + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Cassandra table name for retrieval + */ + @Override + public String getSSTable(String project, List entityNames) { + String tableName = String.format("%s__%s", project, String.join("__", entityNames)); + return trimAndHash(tableName, MAX_TABLE_NAME_LENGTH); + } + + /** + * Converts Cassandra rows into @NativeFeature type. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + @Override + public List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences) { + + BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); + + return rowKeys.stream() + .map( + rowKey -> { + if (!rows.containsKey(rowKey)) { + return Collections.emptyList(); + } else { + Row row = rows.get(rowKey); + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .flatMap( + featureTableColumn -> { + ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); + ByteBuffer schemaRefKey = + row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); + + // Prevent retrieval of features from incorrect FeatureTable + List localFeatureReferences = + featureReferences.stream() + .filter( + featureReference -> + featureReference + .getFeatureTable() + .equals(featureTableColumn)) + .collect(Collectors.toList()); + + List features; + try { + features = + decodeFeatures( + schemaRefKey, + featureValues, + localFeatureReferences, + reusedDecoder, + row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from Cassandra"); + } + + return features.stream(); + }) + .collect(Collectors.toList()); + } + }) + .collect(Collectors.toList()); + } + + /** + * Retrieve rows for each row entity key by generating Cassandra Query with filters based on + * columns. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + List schemaRefColumns = + columnFamilies.stream().map(c -> c + SCHEMA_REF_SUFFIX).collect(Collectors.toList()); + Select query = + QueryBuilder.selectFrom(tableName) + .columns(columnFamilies) + .columns(schemaRefColumns) + .column(ENTITY_KEY); + for (String columnFamily : columnFamilies) { + query = query.writeTime(columnFamily).as(columnFamily + EVENT_TIMESTAMP_SUFFIX); + } + query = query.whereColumn(ENTITY_KEY).isEqualTo(QueryBuilder.bindMarker()); + + PreparedStatement preparedStatement = session.prepare(query.build()); + + List> completableAsyncResultSets = + rowKeys.stream() + .map(preparedStatement::bind) + .map(session::executeAsync) + .map(CompletionStage::toCompletableFuture) + .collect(Collectors.toList()); + + CompletableFuture allResultComputed = + CompletableFuture.allOf(completableAsyncResultSets.toArray(new CompletableFuture[0])); + + Map resultMap; + try { + resultMap = + allResultComputed + .thenApply( + v -> + completableAsyncResultSets.stream() + .map(CompletableFuture::join) + .filter(result -> result.remaining() != 0) + .map(AsyncPagingIterable::one) + .filter(Objects::nonNull) + .collect( + Collectors.toMap( + (Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity()))) + .get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e.getMessage()); + } + + return resultMap; + } + + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. + * + * @param schemaRefKey Schema reference key + * @param value Value of Cassandra cell where bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in Cassandra cell + * @throws IOException + */ + private List decodeFeatures( + ByteBuffer schemaRefKey, + ByteBuffer value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + + if (value == null || schemaRefKey == null) { + return Collections.emptyList(); + } + + CassandraSchemaRegistry.SchemaReference schemaReference = + new CassandraSchemaRegistry.SchemaReference(schemaRefKey); + + // Convert ByteBuffer to ByteArray + byte[] bytesArray = new byte[value.remaining()]; + value.get(bytesArray, 0, bytesArray.length); + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + reusedDecoder = DecoderFactory.get().binaryDecoder(bytesArray, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new AvroFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java new file mode 100644 index 0000000..7915b37 --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; + +public class CassandraSchemaRegistry { + private final CqlSession session; + private final PreparedStatement preparedStatement; + private final LoadingCache> cache; + + private static String SCHEMA_REF_TABLE = "feast_schema_reference"; + private static String SCHEMA_REF_COLUMN = "schema_ref"; + private static String SCHEMA_COLUMN = "avro_schema"; + + public static class SchemaReference { + private final ByteBuffer schemaHash; + + public SchemaReference(ByteBuffer schemaHash) { + this.schemaHash = schemaHash; + } + + public ByteBuffer getSchemaHash() { + return schemaHash; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SchemaReference that = (SchemaReference) o; + return Objects.equals(schemaHash, that.schemaHash); + } + + @Override + public int hashCode() { + return Objects.hash(schemaHash); + } + } + + public CassandraSchemaRegistry(CqlSession session) { + this.session = session; + String tableName = String.format("\"%s\"", SCHEMA_REF_TABLE); + Select query = + QueryBuilder.selectFrom(tableName) + .column(SCHEMA_COLUMN) + .whereColumn(SCHEMA_REF_COLUMN) + .isEqualTo(QueryBuilder.bindMarker()); + this.preparedStatement = session.prepare(query.build()); + + CacheLoader> schemaCacheLoader = + CacheLoader.from(this::loadReader); + + cache = CacheBuilder.newBuilder().build(schemaCacheLoader); + } + + public GenericDatumReader getReader(SchemaReference reference) { + GenericDatumReader reader; + try { + reader = this.cache.get(reference); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { + throw new RuntimeException("Unable to find Schema"); + } + return reader; + } + + private GenericDatumReader loadReader(SchemaReference reference) { + BoundStatement statement = preparedStatement.bind(reference.getSchemaHash()); + + Row row = session.execute(statement).one(); + + Schema schema = + new Schema.Parser() + .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); + return new GenericDatumReader<>(schema); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java new file mode 100644 index 0000000..3ee25df --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +public class CassandraStoreConfig { + + private final String connectionString; + private final String dataCenter; + private final String keySpace; + + public CassandraStoreConfig(String connectionString, String dataCenter, String keySpace) { + this.connectionString = connectionString; + this.dataCenter = dataCenter; + this.keySpace = keySpace; + } + + public String getConnectionString() { + return this.connectionString; + } + + public String getDataCenter() { + return this.dataCenter; + } + + public String getKeySpace() { + return this.keySpace; + } +} diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index 4969364..1c4b75a 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -16,6 +16,9 @@ redis + bigtable + cassandra + sstable diff --git a/storage/connectors/redis/pom.xml b/storage/connectors/redis/pom.xml index bbda8da..f65cbd0 100644 --- a/storage/connectors/redis/pom.xml +++ b/storage/connectors/redis/pom.xml @@ -16,6 +16,14 @@ io.lettuce lettuce-core + 6.0.2.RELEASE + + + + io.netty + netty-transport-native-epoll + 4.1.52.Final + linux-x86_64 diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index 44f74d3..e24b3bd 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -22,6 +22,7 @@ import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.ProtoFeature; import io.lettuce.core.KeyValue; import java.nio.charset.StandardCharsets; import java.util.*; @@ -33,17 +34,17 @@ public class RedisHashDecoder { * * @param redisHashValues retrieved Redis Hash values based on EntityRows * @param byteToFeatureReferenceMap map to decode bytes back to FeatureReference + * @param timestampPrefix timestamp prefix * @return List of {@link Feature} - * @throws InvalidProtocolBufferException + * @throws InvalidProtocolBufferException if a protocol buffer exception occurs */ public static List retrieveFeature( List> redisHashValues, - Map byteToFeatureReferenceMap, + Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { List allFeatures = new ArrayList<>(); - Map allFeaturesBuilderMap = - new HashMap<>(); + HashMap featureMap = new HashMap<>(); Map featureTableTimestampMap = new HashMap<>(); for (KeyValue entity : redisHashValues) { @@ -57,23 +58,22 @@ public static List retrieveFeature( featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); } else { ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK.toString()); + byteToFeatureReferenceMap.get(redisValueK); ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); - Feature.Builder featureBuilder = - Feature.builder().setFeatureReference(featureReference).setFeatureValue(featureValue); - allFeaturesBuilderMap.put(featureReference, featureBuilder); + featureMap.put(featureReference, featureValue); } } } // Add timestamp to features - for (Map.Entry entry : - allFeaturesBuilderMap.entrySet()) { + for (Map.Entry entry : + featureMap.entrySet()) { String timestampRedisHashKeyStr = timestampPrefix + ":" + entry.getKey().getFeatureTable(); Timestamp curFeatureTimestamp = featureTableTimestampMap.get(timestampRedisHashKeyStr); - Feature curFeature = entry.getValue().setEventTimestamp(curFeatureTimestamp).build(); + ProtoFeature curFeature = + new ProtoFeature(entry.getKey(), curFeatureTimestamp, entry.getValue()); allFeatures.add(curFeature); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java similarity index 70% rename from storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java rename to storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java index b79e158..6220dd2 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java @@ -1,6 +1,6 @@ /* * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors + * Copyright 2018-2021 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.redis.serializer; +package feast.storage.connectors.redis.retriever; -import feast.proto.storage.RedisProto.RedisKeyV2; +import feast.proto.storage.RedisProto; -public interface RedisKeySerializerV2 { - - byte[] serialize(RedisKeyV2 key); +@FunctionalInterface +public interface EntityKeySerializer { + byte[] serialize(final RedisProto.RedisKeyV2 entityKey); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java new file mode 100644 index 0000000..922a09d --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import com.google.common.primitives.UnsignedBytes; +import com.google.protobuf.ProtocolStringList; +import feast.proto.storage.RedisProto; +import feast.proto.types.ValueProto; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; + +// This is derived from +// https://github.com/feast-dev/feast/blob/b1ccf8dd1535f721aee8bea937ee38feff80bec5/sdk/python/feast/infra/key_encoding_utils.py#L22 +// and must be kept up to date with any changes in that logic. +public class EntityKeySerializerV2 implements EntityKeySerializer { + + @Override + public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { + final ProtocolStringList joinKeys = entityKey.getEntityNamesList(); + final List values = entityKey.getEntityValuesList(); + + assert joinKeys.size() == values.size(); + + final List buffer = new ArrayList<>(); + + final List> tuples = new ArrayList<>(joinKeys.size()); + for (int i = 0; i < joinKeys.size(); i++) { + tuples.add(Pair.of(joinKeys.get(i), values.get(i))); + } + tuples.sort(Comparator.comparing(Pair::getLeft)); + + ByteBuffer stringBytes = ByteBuffer.allocate(Integer.BYTES); + stringBytes.order(ByteOrder.LITTLE_ENDIAN); + stringBytes.putInt(ValueProto.ValueType.Enum.STRING.getNumber()); + + for (Pair pair : tuples) { + for (final byte b : stringBytes.array()) { + buffer.add(b); + } + for (final byte b : pair.getLeft().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + } + + for (Pair pair : tuples) { + final ValueProto.Value val = pair.getRight(); + switch (val.getValCase()) { + case STRING_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.STRING.getNumber())); + buffer.add( + UnsignedBytes.checkedCast( + val.getStringVal().getBytes(StandardCharsets.UTF_8).length)); + for (final byte b : val.getStringVal().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + break; + case BYTES_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.BYTES.getNumber())); + for (final byte b : val.getBytesVal().toByteArray()) { + buffer.add(b); + } + break; + case INT32_VAL: + ByteBuffer int32ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int32ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int32ByteBuffer.putInt(ValueProto.ValueType.Enum.INT32.getNumber()); + int32ByteBuffer.putInt(Integer.BYTES); + int32ByteBuffer.putInt(val.getInt32Val()); + for (final byte b : int32ByteBuffer.array()) { + buffer.add(b); + } + break; + case INT64_VAL: + ByteBuffer int64ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int64ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int64ByteBuffer.putInt(ValueProto.ValueType.Enum.INT64.getNumber()); + int64ByteBuffer.putInt(Integer.BYTES); + /* This is super dumb - but in https://github.com/feast-dev/feast/blob/dcae1606f53028ce5413567fb8b66f92cfef0f8e/sdk/python/feast/infra/key_encoding_utils.py#L9 + we use `struct.pack("> getOnlineFeatures( String project, List entityRows, - List featureReferences) { + List featureReferences, + List entityNames) { List redisKeys = RedisKeyGenerator.buildRedisKeys(project, entityRows); List> features = getFeaturesFromRedis(redisKeys, featureReferences); @@ -57,11 +63,11 @@ private List> getFeaturesFromRedis( List featureReferences) { List> features = new ArrayList<>(); // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); + Map byteToFeatureReferenceMap = new HashMap<>(); // Serialize using proto List binaryRedisKeys = - redisKeys.stream().map(redisKey -> redisKey.toByteArray()).collect(Collectors.toList()); + redisKeys.stream().map(this.keySerializer::serialize).collect(Collectors.toList()); List featureReferenceWithTsByteList = new ArrayList<>(); featureReferences.stream() @@ -72,7 +78,7 @@ private List> getFeaturesFromRedis( byte[] featureReferenceBytes = RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); + byteToFeatureReferenceMap.put(featureReferenceBytes, featureReference); // eg. <_ts:featuretable_name> byte[] featureTableTsBytes = @@ -96,6 +102,7 @@ private List> getFeaturesFromRedis( future -> { try { List> redisValuesList = future.get(); + List curRedisKeyFeatures = RedisHashDecoder.retrieveFeature( redisValuesList, byteToFeatureReferenceMap, timestampPrefix); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java index 5a7f4b7..e1699bb 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java @@ -16,7 +16,6 @@ */ package feast.storage.connectors.redis.retriever; -import feast.proto.core.StoreProto; import io.lettuce.core.KeyValue; import io.lettuce.core.RedisFuture; import io.lettuce.core.RedisURI; @@ -46,13 +45,18 @@ private RedisClient(StatefulRedisConnection connection) { this.asyncCommands.setAutoFlushCommands(false); } - public static RedisClientAdapter create(StoreProto.Store.RedisConfig config) { + public static RedisClientAdapter create(RedisStoreConfig config) { RedisURI uri = RedisURI.create(config.getHost(), config.getPort()); if (config.getSsl()) { uri.setSsl(true); } + + if (!config.getPassword().isEmpty()) { + uri.setPassword(config.getPassword()); + } + StatefulRedisConnection connection = io.lettuce.core.RedisClient.create(uri).connect(new ByteArrayCodec()); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java index aeb8220..e5bad29 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java @@ -16,36 +16,28 @@ */ package feast.storage.connectors.redis.retriever; -import com.google.common.collect.ImmutableMap; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializerV2; -import feast.storage.connectors.redis.serializer.RedisKeySerializerV2; -import io.lettuce.core.KeyValue; -import io.lettuce.core.ReadFrom; -import io.lettuce.core.RedisFuture; -import io.lettuce.core.RedisURI; +import io.lettuce.core.*; +import io.lettuce.core.cluster.ClusterClientOptions; +import io.lettuce.core.cluster.ClusterTopologyRefreshOptions; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.resource.DnsResolvers; +import io.lettuce.core.resource.MappingSocketAddressResolver; +import io.lettuce.core.resource.NettyCustomizer; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.epoll.EpollChannelOption; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import javax.annotation.Nullable; public class RedisClusterClient implements RedisClientAdapter { private final RedisAdvancedClusterAsyncCommands asyncCommands; - private final RedisKeySerializerV2 serializer; - @Nullable private final RedisKeySerializerV2 fallbackSerializer; - - private static final Map PROTO_TO_LETTUCE_TYPES = - ImmutableMap.of( - RedisClusterConfig.ReadFrom.MASTER, ReadFrom.MASTER, - RedisClusterConfig.ReadFrom.MASTER_PREFERRED, ReadFrom.MASTER_PREFERRED, - RedisClusterConfig.ReadFrom.REPLICA, ReadFrom.REPLICA, - RedisClusterConfig.ReadFrom.REPLICA_PREFERRED, ReadFrom.REPLICA_PREFERRED); @Override public RedisFuture>> hmget(byte[] key, byte[]... fields) { @@ -59,19 +51,9 @@ public void flushCommands() { static class Builder { private final StatefulRedisClusterConnection connection; - private final RedisKeySerializerV2 serializer; - @Nullable private RedisKeySerializerV2 fallbackSerializer; - Builder( - StatefulRedisClusterConnection connection, - RedisKeySerializerV2 serializer) { + Builder(StatefulRedisClusterConnection connection) { this.connection = connection; - this.serializer = serializer; - } - - Builder withFallbackSerializer(RedisKeySerializerV2 fallbackSerializer) { - this.fallbackSerializer = fallbackSerializer; - return this; } RedisClusterClient build() { @@ -81,8 +63,6 @@ RedisClusterClient build() { private RedisClusterClient(Builder builder) { this.asyncCommands = builder.connection.async(); - this.serializer = builder.serializer; - this.fallbackSerializer = builder.fallbackSerializer; // allows reading from replicas this.asyncCommands.readOnly(); @@ -91,31 +71,94 @@ private RedisClusterClient(Builder builder) { this.asyncCommands.setAutoFlushCommands(false); } - public static RedisClientAdapter create(StoreProto.Store.RedisClusterConfig config) { + public static String getAddressString(String host) { + try { + return InetAddress.getByName(host).getHostAddress(); + } catch (UnknownHostException e) { + throw new RuntimeException(String.format("getAllByName() failed: %s", e.getMessage())); + } + } + + public static MappingSocketAddressResolver customSocketAddressResolver( + RedisClusterStoreConfig config) { + + List configuredHosts = + Arrays.stream(config.getConnectionString().split(",")) + .map( + hostPort -> { + return hostPort.trim().split(":")[0]; + }) + .collect(Collectors.toList()); + + Map mapAddressHost = + configuredHosts.stream() + .collect( + Collectors.toMap(host -> ((String) getAddressString(host)), host -> (String) host)); + + return MappingSocketAddressResolver.create( + DnsResolvers.UNRESOLVED, + hostAndPort -> + mapAddressHost.keySet().stream().anyMatch(i -> i.equals(hostAndPort.getHostText())) + ? hostAndPort.of( + mapAddressHost.get(hostAndPort.getHostText()), hostAndPort.getPort()) + : hostAndPort); + } + + public static ClientResources customClientResources(RedisClusterStoreConfig config) { + ClientResources clientResources = + ClientResources.builder() + .nettyCustomizer( + new NettyCustomizer() { + @Override + public void afterBootstrapInitialized(Bootstrap bootstrap) { + bootstrap.option(EpollChannelOption.TCP_KEEPIDLE, 15); + bootstrap.option(EpollChannelOption.TCP_KEEPINTVL, 5); + bootstrap.option(EpollChannelOption.TCP_KEEPCNT, 3); + // Socket Timeout (milliseconds) + bootstrap.option(EpollChannelOption.TCP_USER_TIMEOUT, 60000); + } + }) + .socketAddressResolver(customSocketAddressResolver(config)) + .build(); + return clientResources; + } + + public static RedisClientAdapter create(RedisClusterStoreConfig config) { + List redisURIList = Arrays.stream(config.getConnectionString().split(",")) .map( hostPort -> { String[] hostPortSplit = hostPort.trim().split(":"); - return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); + RedisURI redisURI = + RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); + if (!config.getPassword().isEmpty()) { + redisURI.setPassword(config.getPassword()); + } + if (config.getSsl()) { + redisURI.setSsl(true); + } + return redisURI; }) .collect(Collectors.toList()); - StatefulRedisClusterConnection connection = - io.lettuce.core.cluster.RedisClusterClient.create(redisURIList) - .connect(new ByteArrayCodec()); - - connection.setReadFrom(PROTO_TO_LETTUCE_TYPES.get(config.getReadFrom())); - RedisKeySerializerV2 serializer = new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); + io.lettuce.core.cluster.RedisClusterClient client = + io.lettuce.core.cluster.RedisClusterClient.create( + customClientResources(config), redisURIList); - Builder builder = new Builder(connection, serializer); + client.setOptions( + ClusterClientOptions.builder() + .socketOptions(SocketOptions.builder().keepAlive(true).tcpNoDelay(true).build()) + .timeoutOptions(TimeoutOptions.enabled(config.getTimeout())) + .pingBeforeActivateConnection(true) + .topologyRefreshOptions( + ClusterTopologyRefreshOptions.builder().enableAllAdaptiveRefreshTriggers().build()) + .build()); - if (config.getEnableFallback()) { - RedisKeySerializerV2 fallbackSerializer = - new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); - builder = builder.withFallbackSerializer(fallbackSerializer); - } + StatefulRedisClusterConnection connection = + client.connect(new ByteArrayCodec()); + connection.setReadFrom(config.getReadFrom()); - return builder.build(); + return new Builder(connection).build(); } } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java new file mode 100644 index 0000000..a7278bd --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterStoreConfig.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import io.lettuce.core.ReadFrom; +import java.time.Duration; + +public class RedisClusterStoreConfig { + private final String connectionString; + private final ReadFrom readFrom; + private final Duration timeout; + private final String password; + private final Boolean ssl; + + public RedisClusterStoreConfig( + String connectionString, ReadFrom readFrom, Duration timeout, Boolean ssl, String password) { + this.connectionString = connectionString; + this.readFrom = readFrom; + this.timeout = timeout; + this.password = password; + this.ssl = ssl; + } + + public String getConnectionString() { + return this.connectionString; + } + + public ReadFrom getReadFrom() { + return this.readFrom; + } + + public Duration getTimeout() { + return this.timeout; + } + + public String getPassword() { + return this.password; + } + + public Boolean getSsl() { + return this.ssl; + } +} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java new file mode 100644 index 0000000..3045235 --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisStoreConfig.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +public class RedisStoreConfig { + private final String host; + private final Integer port; + private final Boolean ssl; + private final String password; + + public RedisStoreConfig(String host, Integer port, Boolean ssl, String password) { + this.host = host; + this.port = port; + this.ssl = ssl; + this.password = password; + } + + public String getHost() { + return this.host; + } + + public Integer getPort() { + return this.port; + } + + public Boolean getSsl() { + return this.ssl; + } + + public String getPassword() { + return this.password; + } +} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java deleted file mode 100644 index 1c869b4..0000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKeyV2; - -public class RedisKeyPrefixSerializerV2 implements RedisKeySerializerV2 { - - private final byte[] prefixBytes; - - public RedisKeyPrefixSerializerV2(String prefix) { - this.prefixBytes = prefix.getBytes(); - } - - public byte[] serialize(RedisKeyV2 redisKey) { - byte[] key = redisKey.toByteArray(); - - if (prefixBytes.length == 0) { - return key; - } - - byte[] keyWithPrefix = new byte[prefixBytes.length + key.length]; - System.arraycopy(prefixBytes, 0, keyWithPrefix, 0, prefixBytes.length); - System.arraycopy(key, 0, keyWithPrefix, prefixBytes.length, key.length); - return keyWithPrefix; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java deleted file mode 100644 index 252d6d1..0000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKeyV2; - -public class RedisKeyProtoSerializerV2 implements RedisKeySerializerV2 { - - public byte[] serialize(RedisKeyV2 redisKey) { - return redisKey.toByteArray(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java deleted file mode 100644 index e663cf8..0000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import static org.junit.Assert.*; - -import feast.proto.storage.RedisProto.RedisKeyV2; -import feast.proto.types.ValueProto; -import org.junit.Test; - -public class RedisKeyPrefixSerializerTest { - - private RedisKeyV2 key = - RedisKeyV2.newBuilder() - .addEntityNames("entity1") - .addEntityValues(ValueProto.Value.newBuilder().setInt64Val(1)) - .build(); - - @Test - public void shouldPrependKey() { - RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2("namespace:"); - String keyWithPrefix = new String(serializer.serialize(key)); - assertEquals(String.format("namespace:%s", new String(key.toByteArray())), keyWithPrefix); - } - - @Test - public void shouldNotPrependKeyIfEmptyString() { - RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2(""); - assertArrayEquals(key.toByteArray(), serializer.serialize(key)); - } -} diff --git a/storage/connectors/sstable/pom.xml b/storage/connectors/sstable/pom.xml new file mode 100644 index 0000000..8c7a271 --- /dev/null +++ b/storage/connectors/sstable/pom.xml @@ -0,0 +1,19 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-sstable + + + 11 + 11 + + + \ No newline at end of file diff --git a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java new file mode 100644 index 0000000..c86923a --- /dev/null +++ b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.sstable.retriever; + +import com.google.common.hash.Hashing; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; +import feast.proto.types.ValueProto; +import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.OnlineRetrieverV2; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @param Decoded value type of the partition key + * @param Type of the SSTable row + */ +public interface SSTableOnlineRetriever extends OnlineRetrieverV2 { + + int MAX_TABLE_NAME_LENGTH = 50; + + @Override + default List> getOnlineFeatures( + String project, + List entityRows, + List featureReferences, + List entityNames) { + + List columnFamilies = getSSTableColumns(featureReferences); + String tableName = getSSTable(project, entityNames); + + List rowKeys = + entityRows.stream() + .map(row -> convertEntityValueToKey(row, entityNames)) + .collect(Collectors.toList()); + + Map rowsFromSSTable = getFeaturesFromSSTable(tableName, rowKeys, columnFamilies); + + return convertRowToFeature(tableName, rowKeys, rowsFromSSTable, featureReferences); + } + + /** + * Generate SSTable key. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return SSTable key for retrieval + */ + K convertEntityValueToKey(EntityRow entityRow, List entityNames); + + /** + * Converts SSTable rows into @NativeFeature type. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences); + + /** + * Retrieve rows for each row entity key. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of column names + * @return Map of retrieved features for each rowKey + */ + Map getFeaturesFromSSTable(String tableName, List rowKeys, List columnFamilies); + + /** + * Retrieve name of SSTable corresponding to entities in retrieval call + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Name of Cassandra table + */ + default String getSSTable(String project, List entityNames) { + return trimAndHash( + String.format("%s__%s", project, String.join("__", entityNames)), MAX_TABLE_NAME_LENGTH); + } + + /** + * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, + * INT64_VAL, INT32_VAL and BYTES_VAL. + * + * @param v Entity value of Feast valueType + * @return String representation of Entity value + */ + default String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + + /** + * Retrieve SSTable columns based on Feature references. + * + * @param featureReferences List of feature references in retrieval call + * @return List of String of column names + */ + default List getSSTableColumns(List featureReferences) { + return featureReferences.stream() + .map(FeatureReferenceV2::getFeatureTable) + .distinct() + .collect(Collectors.toList()); + } + + /** + * Trims long SSTable table names and appends hash suffix for uniqueness. + * + * @param expr Original SSTable table name + * @param maxLength Maximum length allowed for SSTable + * @return Hashed suffix SSTable table name + */ + default String trimAndHash(String expr, int maxLength) { + // Length 8 as derived from murmurhash_32 implementation + int maxPrefixLength = maxLength - 8; + String finalName = expr; + if (expr.length() > maxLength) { + String hashSuffix = + Hashing.murmur3_32().hashBytes(expr.substring(maxPrefixLength).getBytes()).toString(); + finalName = expr.substring(0, Math.min(expr.length(), maxPrefixLength)).concat(hashSuffix); + } + return finalName; + } +}