diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 660304a8546..ebe0216a0f6 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -3,45 +3,6 @@ name: complete on: [push, pull_request] jobs: - build-push-docker-images: - runs-on: [self-hosted] - strategy: - matrix: - component: [core, serving, jupyter, ci] - env: - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} - REGISTRY: gcr.io/kf-feast - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - steps: - - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - - name: Build image - run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - - name: Push image - run: | - docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA} - if [ -n "${GITHUB_PR_SHA}" ]; then - docker tag ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_PR_SHA} - docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_PR_SHA} - fi - - lint-java: - container: gcr.io/kf-feast/feast-ci:latest - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: Lint java - run: make lint-java - lint-python: container: gcr.io/kf-feast/feast-ci:latest runs-on: [ubuntu-latest] @@ -64,37 +25,6 @@ jobs: - name: Lint go run: make lint-go - lint-versions: - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: Lint versions throughout repo - run: make lint-versions - - unit-test-java: - runs-on: ubuntu-latest - needs: lint-java - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-ut-maven- - - name: Test java - run: make test-java-with-coverage - - uses: actions/upload-artifact@v2 - with: - name: java-coverage-report - path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ - unit-test-python: runs-on: ubuntu-latest needs: lint-python @@ -117,33 +47,3 @@ jobs: - name: Test go run: make test-go - integration-test: - runs-on: ubuntu-latest - needs: unit-test-java - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - architecture: 'x64' - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-it-maven- - - name: Run integration tests - run: make test-java-integration - - name: Save report - uses: actions/upload-artifact@v2 - if: failure() - with: - name: it-report - path: spark/ingestion/target/test-reports/TestSuite.txt - retention-days: 5 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml deleted file mode 100644 index dadc1c27f9d..00000000000 --- a/.github/workflows/master_only.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: master only - -on: - push: - branches: master - tags: - - 'v*.*.*' - -jobs: - build-docker-images: - runs-on: [self-hosted] - strategy: - matrix: - component: [core, serving, jupyter, ci] - env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - steps: - - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - - name: Get version - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Build image - run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} - - name: Push image - run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} - - name: Push development Docker image - run: | - if [ ${GITHUB_REF#refs/*/} == "master" ]; then - docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop - docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop - fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 157a593d35f..1622995f1b6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: needs: get-version strategy: matrix: - component: [core, serving, jobservice, jupyter] + component: [jobservice, jupyter] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar steps: @@ -111,8 +111,6 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - - name: Validate repository versions - run: make lint-versions - name: Validate chart release versions run: ./infra/scripts/validate-helm-chart-docker-image.sh - name: Remove previous Helm @@ -120,4 +118,4 @@ jobs: - name: Install Helm run: ./infra/scripts/install-helm.sh - name: Publish Helm charts - run: ./infra/scripts/sync-helm-charts.sh \ No newline at end of file + run: ./infra/scripts/sync-helm-charts.sh diff --git a/.scalafmt.conf b/.scalafmt.conf deleted file mode 100644 index f3c72b8ceb5..00000000000 --- a/.scalafmt.conf +++ /dev/null @@ -1,2 +0,0 @@ -align.preset = more -maxColumn = 100 \ No newline at end of file diff --git a/Makefile b/Makefile index 89b9433bfa6..9b55c9d1111 100644 --- a/Makefile +++ b/Makefile @@ -17,47 +17,20 @@ ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) PROTO_TYPE_SUBDIRS = core serving types storage PROTO_SERVICE_SUBDIRS = core serving -MVN := mvn ${MAVEN_EXTRA_OPTS} # General -format: format-python format-go format-java +format: format-python format-go -lint: lint-python lint-go lint-java +lint: lint-python lint-go -test: test-python test-java test-go +test: test-python test-go protos: compile-protos-go compile-protos-python compile-protos-docs -build: protos build-java build-docker build-html +build: protos build-docker build-html -install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependencies install-java-ci-dependencies - -# Java - -install-java-ci-dependencies: - ${MVN} verify clean --fail-never - -format-java: - ${MVN} spotless:apply - -lint-java: - ${MVN} --no-transfer-progress spotless:check - -test-java: - ${MVN} --no-transfer-progress -DskipITs=true test - -test-java-integration: - ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean verify - -test-java-with-coverage: - ${MVN} --no-transfer-progress -DskipITs=true test jacoco:report-aggregate - -build-java: - ${MVN} clean verify - -build-java-no-tests: - ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -DskipITs=true -Drevision=${REVISION} clean package +install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependencies # Python SDK @@ -112,17 +85,9 @@ lint-go: build-push-docker: @$(MAKE) build-docker registry=$(REGISTRY) version=$(VERSION) - @$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION) - @$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION) @$(MAKE) push-ci-docker registry=$(REGISTRY) version=$(VERSION) -build-docker: build-core-docker build-serving-docker build-ci-docker - -push-core-docker: - docker push $(REGISTRY)/feast-core:$(VERSION) - -push-serving-docker: - docker push $(REGISTRY)/feast-serving:$(VERSION) +build-docker: build-ci-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) @@ -130,12 +95,6 @@ push-ci-docker: push-jupyter-docker: docker push $(REGISTRY)/feast-jupyter:$(VERSION) -build-core-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . - -build-serving-docker: - docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . - build-ci-docker: docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile . @@ -183,11 +142,6 @@ build-html: clean-html cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) html cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python -# Versions - -lint-versions: - ./infra/scripts/validate-version-consistency.sh - # Performance test-load: diff --git a/common-test/pom.xml b/common-test/pom.xml deleted file mode 100644 index 0232b4dc7f1..00000000000 --- a/common-test/pom.xml +++ /dev/null @@ -1,172 +0,0 @@ - - - - 4.0.0 - - - feast-parent - dev.feast - ${revision} - - - Feast Common Test - Feast common module with test utilities - feast-common-test - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true - - - - - - - - dev.feast - datatypes-java - ${project.version} - compile - - - dev.feast - feast-common - ${project.version} - compile - - - com.google.protobuf - protobuf-java-util - - - - org.projectlombok - lombok - ${lombok.version} - - - javax.validation - validation-api - - - com.google.auto.value - auto-value-annotations - - - com.google.code.gson - gson - - - - - org.slf4j - slf4j-api - - - - org.hamcrest - hamcrest-library - - - org.springframework.boot - spring-boot-test - ${spring.boot.version} - - - org.springframework.boot - spring-boot-test-autoconfigure - - - org.springframework - spring-test - ${spring.version} - - - org.testcontainers - junit-jupiter - 1.15.1 - - - org.testcontainers - postgresql - 1.15.1 - - - org.testcontainers - kafka - 1.15.1 - - - org.junit.jupiter - junit-jupiter-api - 5.6.2 - - - org.springframework.kafka - spring-kafka - - - io.prometheus - simpleclient - 0.8.0 - - - org.apache.commons - commons-lang3 - 3.4 - - - org.awaitility - awaitility - 3.0.0 - - - org.awaitility - awaitility-proxy - 3.0.0 - - - org.mockito - mockito-core - ${mockito.version} - compile - - - io.rest-assured - rest-assured - 4.2.0 - - - io.rest-assured - json-path - 4.2.0 - - - io.rest-assured - xml-path - 4.2.0 - - - diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java deleted file mode 100644 index f82a804ee1c..00000000000 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import io.prometheus.client.CollectorRegistry; -import java.sql.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.junit.jupiter.api.*; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Base Integration Test class. Setups postgres and kafka containers. Configures related properties - * and beans. Provides DB related clean up between tests. - */ -@SpringBootTest -@ActiveProfiles("it") -@Testcontainers -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public class BaseIT { - - @Container public static PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer<>(); - - @Container public static KafkaContainer kafka = new KafkaContainer(); - - /** - * Configure Spring Application to use postgres and kafka rolled out in containers - * - * @param registry - */ - @DynamicPropertySource - static void properties(DynamicPropertyRegistry registry) { - - registry.add("spring.datasource.url", postgreSQLContainer::getJdbcUrl); - registry.add("spring.datasource.username", postgreSQLContainer::getUsername); - registry.add("spring.datasource.password", postgreSQLContainer::getPassword); - registry.add("spring.jpa.hibernate.ddl-auto", () -> "none"); - - registry.add("feast.stream.options.bootstrapServers", kafka::getBootstrapServers); - } - - /** - * SequentialFlow is base class that is supposed to be inherited by @Nested test classes that - * wants to preserve context between test cases. For SequentialFlow databases is being truncated - * only once after all tests passed. - */ - @TestInstance(TestInstance.Lifecycle.PER_CLASS) - public class SequentialFlow { - @AfterAll - public void tearDown() throws Exception { - cleanTables(); - } - } - - /** - * This class must be inherited inside IT Class and annotated with {@link - * org.springframework.boot.test.context.TestConfiguration}. It provides configuration needed to - * communicate with Feast via Kafka - */ - public static class BaseTestConfig { - @Bean - public KafkaListenerContainerFactory> - testListenerContainerFactory(ConsumerFactory consumerFactory) { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - @Bean - public ConsumerFactory testConsumerFactory() { - Map props = new HashMap<>(); - - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, this.getClass().getName()); - - return new DefaultKafkaConsumerFactory<>( - props, new StringDeserializer(), new ByteArrayDeserializer()); - } - } - - /** - * Truncates all tables in Database (between tests or flows). Retries on deadlock - * - * @throws SQLException - */ - public static void cleanTables() throws SQLException { - Connection connection = - DriverManager.getConnection( - postgreSQLContainer.getJdbcUrl(), - postgreSQLContainer.getUsername(), - postgreSQLContainer.getPassword()); - - List tableNames = new ArrayList<>(); - Statement statement = connection.createStatement(); - ResultSet rs = - statement.executeQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema='public'"); - while (rs.next()) { - tableNames.add(rs.getString(1)); - } - - if (tableNames.isEmpty()) { - return; - } - - // retries are needed since truncate require exclusive lock - // and that often leads to Deadlock - // since SpringApp is still running in another thread - int num_retries = 5; - for (int i = 1; i <= num_retries; i++) { - try { - statement = connection.createStatement(); - statement.execute(String.format("truncate %s cascade", String.join(", ", tableNames))); - } catch (SQLException e) { - if (i == num_retries) { - throw e; - } - continue; - } - - break; - } - } - - /** Used to determine SequentialFlows */ - public Boolean isSequentialTest(TestInfo testInfo) { - try { - testInfo.getTestClass().get().asSubclass(SequentialFlow.class); - } catch (ClassCastException e) { - return false; - } - return true; - } - - @AfterEach - public void tearDown(TestInfo testInfo) throws Exception { - CollectorRegistry.defaultRegistry.clear(); - - if (!isSequentialTest(testInfo)) { - cleanTables(); - } - } -} diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java deleted file mode 100644 index 0606c759516..00000000000 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.DataFormatProto.FileFormat; -import feast.proto.core.DataFormatProto.FileFormat.ParquetFormat; -import feast.proto.core.DataFormatProto.StreamFormat; -import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat; -import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat; -import feast.proto.core.DataSourceProto.DataSource; -import feast.proto.core.DataSourceProto.DataSource.FileOptions; -import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto; -import feast.proto.serving.ServingAPIProto; -import feast.proto.types.ValueProto; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Triple; - -public class DataGenerator { - // projectName, featureName, exclude - static Triple defaultSubscription = Triple.of("*", "*", false); - - static StoreProto.Store defaultStore = - createStore( - "test-store", StoreProto.Store.StoreType.REDIS, ImmutableList.of(defaultSubscription)); - - public static Triple getDefaultSubscription() { - return defaultSubscription; - } - - public static StoreProto.Store getDefaultStore() { - return defaultStore; - } - - public static StoreProto.Store createStore( - String name, - StoreProto.Store.StoreType type, - List> subscriptions) { - StoreProto.Store.Builder builder = - StoreProto.Store.newBuilder() - .addAllSubscriptions( - subscriptions.stream() - .map( - s -> - StoreProto.Store.Subscription.newBuilder() - .setProject(s.getLeft()) - .setName(s.getMiddle()) - .setExclude(s.getRight()) - .build()) - .collect(Collectors.toList())) - .setName(name) - .setType(type); - - switch (type) { - case REDIS: - StoreProto.Store.RedisConfig redisConfig = - StoreProto.Store.RedisConfig.newBuilder().build(); - return builder.setRedisConfig(redisConfig).build(); - case REDIS_CLUSTER: - StoreProto.Store.RedisClusterConfig redisClusterConfig = - StoreProto.Store.RedisClusterConfig.newBuilder().build(); - return builder.setRedisClusterConfig(redisClusterConfig).build(); - default: - throw new RuntimeException("Unrecognized Store type"); - } - } - - public static EntityProto.EntitySpecV2 createEntitySpecV2( - String name, - String description, - ValueProto.ValueType.Enum valueType, - Map labels) { - return EntityProto.EntitySpecV2.newBuilder() - .setName(name) - .setDescription(description) - .setValueType(valueType) - .putAllLabels(labels) - .build(); - } - - public static FeatureProto.FeatureSpecV2 createFeatureSpecV2( - String name, ValueProto.ValueType.Enum valueType, Map labels) { - return FeatureProto.FeatureSpecV2.newBuilder() - .setName(name) - .setValueType(valueType) - .putAllLabels(labels) - .build(); - } - - // Create a Feature Table spec without DataSources configured. - public static FeatureTableSpec createFeatureTableSpec( - String name, - List entities, - Map features, - int maxAgeSecs, - Map labels) { - - return FeatureTableSpec.newBuilder() - .setName(name) - .addAllEntities(entities) - .addAllFeatures( - features.entrySet().stream() - .map( - entry -> - FeatureSpecV2.newBuilder() - .setName(entry.getKey()) - .setValueType(entry.getValue()) - .putAllLabels(labels) - .build()) - .collect(Collectors.toList())) - .setMaxAge(Duration.newBuilder().setSeconds(3600).build()) - .setBatchSource( - DataSource.newBuilder() - .setEventTimestampColumn("ts") - .setType(DataSource.SourceType.BATCH_FILE) - .setFileOptions( - FileOptions.newBuilder() - .setFileFormat( - FileFormat.newBuilder() - .setParquetFormat(ParquetFormat.newBuilder().build()) - .build()) - .setFileUrl("/dev/null") - .build()) - .build()) - .putAllLabels(labels) - .build(); - } - - public static FeatureTableSpec createFeatureTableSpec( - String name, - List entities, - ImmutableMap features, - int maxAgeSecs, - Map labels) { - - return FeatureTableSpec.newBuilder() - .setName(name) - .addAllEntities(entities) - .addAllFeatures( - features.entrySet().stream() - .map( - entry -> - FeatureSpecV2.newBuilder() - .setName(entry.getKey()) - .setValueType(entry.getValue()) - .putAllLabels(labels) - .build()) - .collect(Collectors.toList())) - .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSecs).build()) - .putAllLabels(labels) - .build(); - } - - public static DataSource createFileDataSourceSpec( - String fileURL, String timestampColumn, String datePartitionColumn) { - return DataSource.newBuilder() - .setType(DataSource.SourceType.BATCH_FILE) - .setFileOptions( - FileOptions.newBuilder() - .setFileFormat(createParquetFormat()) - .setFileUrl(fileURL) - .build()) - .setEventTimestampColumn(timestampColumn) - .setDatePartitionColumn(datePartitionColumn) - .build(); - } - - public static DataSource createBigQueryDataSourceSpec( - String bigQueryTableRef, String timestampColumn, String datePartitionColumn) { - return DataSource.newBuilder() - .setType(DataSource.SourceType.BATCH_BIGQUERY) - .setBigqueryOptions( - DataSource.BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build()) - .setEventTimestampColumn(timestampColumn) - .setDatePartitionColumn(datePartitionColumn) - .build(); - } - - public static DataSource createKafkaDataSourceSpec( - String servers, String topic, String classPath, String timestampColumn) { - return DataSource.newBuilder() - .setType(DataSource.SourceType.STREAM_KAFKA) - .setKafkaOptions( - KafkaOptions.newBuilder() - .setTopic(topic) - .setBootstrapServers(servers) - .setMessageFormat(createProtoFormat("class.path")) - .build()) - .setEventTimestampColumn(timestampColumn) - .build(); - } - - public static ValueProto.Value createEmptyValue() { - return ValueProto.Value.newBuilder().build(); - } - - public static ValueProto.Value createStrValue(String val) { - return ValueProto.Value.newBuilder().setStringVal(val).build(); - } - - public static ValueProto.Value createDoubleValue(double value) { - return ValueProto.Value.newBuilder().setDoubleVal(value).build(); - } - - public static ValueProto.Value createInt64Value(long value) { - return ValueProto.Value.newBuilder().setInt64Val(value).build(); - } - - public static ServingAPIProto.FeatureReferenceV2 createFeatureReference( - String featureTableName, String featureName) { - return ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(featureTableName) - .setName(featureName) - .build(); - } - - public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityRow( - String entityName, ValueProto.Value entityValue, long seconds) { - return ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(seconds)) - .putFields(entityName, entityValue) - .build(); - } - - public static DataSource createKinesisDataSourceSpec( - String region, String streamName, String classPath, String timestampColumn) { - return DataSource.newBuilder() - .setType(DataSource.SourceType.STREAM_KINESIS) - .setKinesisOptions( - KinesisOptions.newBuilder() - .setRegion("ap-nowhere1") - .setStreamName("stream") - .setRecordFormat(createProtoFormat(classPath)) - .build()) - .setEventTimestampColumn(timestampColumn) - .build(); - } - - public static FileFormat createParquetFormat() { - return FileFormat.newBuilder().setParquetFormat(ParquetFormat.getDefaultInstance()).build(); - } - - public static StreamFormat createAvroFormat(String schemaJSON) { - return StreamFormat.newBuilder() - .setAvroFormat(AvroFormat.newBuilder().setSchemaJson(schemaJSON).build()) - .build(); - } - - public static StreamFormat createProtoFormat(String classPath) { - return StreamFormat.newBuilder() - .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build()) - .build(); - } -} diff --git a/common-test/src/main/java/feast/common/it/ExternalApp.java b/common-test/src/main/java/feast/common/it/ExternalApp.java deleted file mode 100644 index 3fa98b0d502..00000000000 --- a/common-test/src/main/java/feast/common/it/ExternalApp.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import com.google.auto.value.AutoValue; -import java.util.HashMap; -import java.util.Map; -import org.springframework.boot.Banner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.MapPropertySource; -import org.springframework.core.env.StandardEnvironment; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.PostgreSQLContainer; - -@AutoValue -public abstract class ExternalApp { - private ConfigurableApplicationContext appContext; - - abstract PostgreSQLContainer getPostgreSQL(); - - abstract String getName(); - - abstract int getGRPCPort(); - - abstract int getWebPort(); - - abstract Map getProperties(); - - abstract Class getSpringApplication(); - - public static Builder builder() { - return new AutoValue_ExternalApp.Builder() - .setProperties(new HashMap<>()) - .setWebPort(SocketUtils.findAvailableTcpPort()); - } - - @AutoValue.Builder - public interface Builder { - Builder setSpringApplication(Class app); - - Builder setName(String name); - - Builder setPostgreSQL(PostgreSQLContainer psql); - - Builder setGRPCPort(int port); - - Builder setWebPort(int port); - - Builder setProperties(Map properties); - - ExternalApp build(); - } - - public void start() { - HashMap properties = new HashMap<>(getProperties()); - properties.put("spring.datasource.url", getPostgreSQL().getJdbcUrl()); - properties.put("spring.datasource.username", getPostgreSQL().getUsername()); - properties.put("spring.datasource.password", getPostgreSQL().getPassword()); - properties.put("grpc.server.port", getGRPCPort()); - properties.put("server.port", getWebPort()); - - StandardEnvironment env = new StandardEnvironment(); - env.setDefaultProfiles(getName()); - env.getPropertySources().addFirst(new MapPropertySource("primary", properties)); - - appContext = - new SpringApplicationBuilder(getSpringApplication()) - .environment(env) - .bannerMode(Banner.Mode.OFF) - .run(); - } - - public void stop() { - SpringApplication.exit(appContext); - } -} diff --git a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java deleted file mode 100644 index 11fa6715dc8..00000000000 --- a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import feast.proto.core.*; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableRequest; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class SimpleCoreClient { - private CoreServiceGrpc.CoreServiceBlockingStub stub; - - public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { - this.stub = stub; - } - - public CoreServiceProto.ApplyEntityResponse simpleApplyEntity( - String projectName, EntityProto.EntitySpecV2 spec) { - return stub.applyEntity( - CoreServiceProto.ApplyEntityRequest.newBuilder() - .setProject(projectName) - .setSpec(spec) - .build()); - } - - public List simpleListEntities(String projectName) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder() - .setProject(projectName) - .build()) - .build()) - .getEntitiesList(); - } - - public List simpleListEntities( - String projectName, Map labels) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder() - .setProject(projectName) - .putAllLabels(labels) - .build()) - .build()) - .getEntitiesList(); - } - - public List simpleListEntities( - CoreServiceProto.ListEntitiesRequest.Filter filter) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder().setFilter(filter).build()) - .getEntitiesList(); - } - - public List simpleListFeatureTables( - CoreServiceProto.ListFeatureTablesRequest.Filter filter) { - return stub.listFeatureTables( - CoreServiceProto.ListFeatureTablesRequest.newBuilder().setFilter(filter).build()) - .getTablesList(); - } - - public EntityProto.Entity simpleGetEntity(String projectName, String name) { - return stub.getEntity( - CoreServiceProto.GetEntityRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getEntity(); - } - - public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { - return stub.getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getTable(); - } - - public Map simpleListFeatures( - String projectName, Map labels, List entities) { - return stub.listFeatures( - CoreServiceProto.ListFeaturesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeaturesRequest.Filter.newBuilder() - .setProject(projectName) - .addAllEntities(entities) - .putAllLabels(labels) - .build()) - .build()) - .getFeaturesMap(); - } - - public Map simpleListFeatures( - String projectName, String... entities) { - return simpleListFeatures(projectName, Collections.emptyMap(), Arrays.asList(entities)); - } - - public CoreServiceProto.UpdateStoreResponse updateStore(StoreProto.Store store) { - return stub.updateStore( - CoreServiceProto.UpdateStoreRequest.newBuilder().setStore(store).build()); - } - - public void createProject(String name) { - stub.createProject(CoreServiceProto.CreateProjectRequest.newBuilder().setName(name).build()); - } - - public void archiveProject(String name) { - stub.archiveProject(CoreServiceProto.ArchiveProjectRequest.newBuilder().setName(name).build()); - } - - public String getFeastCoreVersion() { - return stub.getFeastCoreVersion( - CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance()) - .getVersion(); - } - - public FeatureTableProto.FeatureTable applyFeatureTable( - String projectName, FeatureTableSpec spec) { - return stub.applyFeatureTable( - ApplyFeatureTableRequest.newBuilder() - .setProject(projectName) - .setTableSpec(spec) - .build()) - .getTable(); - } - - public void deleteFeatureTable(String projectName, String featureTableName) { - stub.deleteFeatureTable( - CoreServiceProto.DeleteFeatureTableRequest.newBuilder() - .setProject(projectName) - .setName(featureTableName) - .build()); - } -} diff --git a/common-test/src/main/java/feast/common/it/SimpleIT.java b/common-test/src/main/java/feast/common/it/SimpleIT.java deleted file mode 100644 index e8c5dff38c9..00000000000 --- a/common-test/src/main/java/feast/common/it/SimpleIT.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest() -public class SimpleIT extends BaseIT { - @Test - public void test() { - assert true; - } -} diff --git a/common-test/src/main/java/feast/common/util/TestUtil.java b/common-test/src/main/java/feast/common/util/TestUtil.java deleted file mode 100644 index ee355d3766f..00000000000 --- a/common-test/src/main/java/feast/common/util/TestUtil.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.util; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import feast.common.logging.AuditLogger; -import feast.common.logging.config.LoggingProperties; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.Comparator; -import java.util.stream.Collectors; -import org.springframework.boot.info.BuildProperties; - -public class TestUtil { - /** Setup the audit logger. This call is required to use the audit logger when testing. */ - public static void setupAuditLogger() { - LoggingProperties.AuditLogProperties properties = new LoggingProperties.AuditLogProperties(); - properties.setEnabled(true); - LoggingProperties loggingProperties = new LoggingProperties(); - loggingProperties.setAudit(properties); - - BuildProperties buildProperties = mock(BuildProperties.class); - when(buildProperties.getArtifact()).thenReturn("feast-core"); - when(buildProperties.getVersion()).thenReturn("0.6"); - - new AuditLogger(loggingProperties, buildProperties); - } - - /** - * Compare if two Feature Table specs are equal. Disregards order of features/entities in spec. - */ - public static boolean compareFeatureTableSpec(FeatureTableSpec spec, FeatureTableSpec otherSpec) { - spec = - spec.toBuilder() - .clearFeatures() - .addAllFeatures( - spec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpecV2::getName)) - .collect(Collectors.toSet())) - .clearEntities() - .addAllEntities(spec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) - .build(); - - otherSpec = - otherSpec - .toBuilder() - .clearFeatures() - .addAllFeatures( - otherSpec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpecV2::getName)) - .collect(Collectors.toSet())) - .clearEntities() - .addAllEntities( - otherSpec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) - .build(); - - return spec.equals(otherSpec); - } -} diff --git a/common/.openapi-generator-ignore b/common/.openapi-generator-ignore deleted file mode 100644 index 6b177032ba1..00000000000 --- a/common/.openapi-generator-ignore +++ /dev/null @@ -1,20 +0,0 @@ -settings.gradle -README.md -pom.xml -gradle -git_push.sh -build.sbt -build.gradle -.travis* -.gitignore -src/main/resources/api.yaml -gradle* -gradle/* -gradle-wrapper.* -gradle** -gradle/ -src/main/java/feast/auth/providers/http/HttpAuthorizationProvider.java -src/main/java/feast/auth/providers/http/ketoadaptor/api/CheckAccessApiController.java -src/main/java/feast/auth/providers/http/ketoadaptor/api/KetoAuth.java -src/main/AndroidManifest.xml -.openapi-generator/ \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml deleted file mode 100644 index ce89e9c6a50..00000000000 --- a/common/pom.xml +++ /dev/null @@ -1,277 +0,0 @@ - - - - 4.0.0 - - - feast-parent - dev.feast - ${revision} - - - Feast Common - Feast common module with functionality that can be reused - feast-common - - - - dev.feast - datatypes-java - ${project.version} - compile - - - com.google.protobuf - protobuf-java-util - - - - org.apache.commons - commons-lang3 - 3.6 - - - - - org.projectlombok - lombok - - - com.google.auto.value - auto-value-annotations - - - - - com.google.code.gson - gson - - - io.gsonfire - gson-fire - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - - - org.springframework - spring-context-support - - - net.devh - grpc-server-spring-boot-starter - - - org.springframework.boot - spring-boot-starter-logging - - - - - org.springframework.boot - spring-boot-starter-data-jpa - - - org.springframework.boot - spring-boot-starter-web - - - org.hibernate.validator - hibernate-validator - - - - - org.springframework.security - spring-security-core - - - org.springframework.security - spring-security-config - - - org.springframework.security - spring-security-oauth2-resource-server - - - org.springframework.security - spring-security-oauth2-jose - - - com.google.auth - google-auth-library-oauth2-http - - - - - org.openapitools - jackson-databind-nullable - 0.1.0 - - - io.swagger - swagger-annotations - - - com.squareup.okhttp3 - okhttp - - - com.squareup.okhttp3 - logging-interceptor - - - io.springfox - springfox-swagger2 - - - io.springfox - springfox-swagger-ui - - - - - org.slf4j - slf4j-api - - - org.fluentd - fluent-logger - 0.3.1 - - - - - javax.xml.bind - jaxb-api - - - javax.validation - validation-api - - - - - com.google.code.findbugs - jsr305 - 3.0.2 - - - - - org.hamcrest - hamcrest-library - test - - - org.apache.kafka - kafka-clients - 2.5.0 - - - junit - junit - 4.12 - - - org.springframework - spring-test - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - org.springframework.boot - spring-boot-starter-test - test - - - org.junit.vintage - junit-vintage-engine - - - - - - - - org.openapitools - openapi-generator-maven-plugin - 4.3.1 - - - client - - generate - - - ${project.basedir}/src/main/resources/api.yaml - java - ${feast.auth.providers.http.client.package.name} - ${feast.auth.providers.http.client.package.name}.model - ${feast.auth.providers.http.client.package.name}.api - ${feast.auth.providers.http.client.package.name}.invoker - - ${project.groupId} - ${project.artifactId} - ${project.version} - true - java8 - Apache 2.0 - https://www.apache.org/licenses/LICENSE-2.0 - ${project.build.directory}/generated-sources - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - ${feast.auth.providers.http.client.package.name}.* - - - - org.jacoco - jacoco-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true - - - - - diff --git a/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java b/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java deleted file mode 100644 index 2b5c89f66e6..00000000000 --- a/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authentication; - -import java.util.Map; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; -import org.springframework.security.oauth2.jwt.NimbusJwtDecoder; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationProvider; - -/** Json Web Token Authentication Provider used to validate incoming requests to Feast Core. */ -public class DefaultJwtAuthenticationProvider implements AuthenticationProvider { - - private JwtAuthenticationProvider authProvider; - - /** - * @param options String K/V pair of options to initialize the AuthenticationProvider with. Only - * one option is currently configurable, the jwkEndpointURI. - */ - public DefaultJwtAuthenticationProvider(Map options) { - // Endpoint used to retrieve certificates to validate JWT token - String jwkEndpointURI = options.get("jwkEndpointURI"); - - // Provide a custom endpoint to retrieve certificates - authProvider = - new JwtAuthenticationProvider(NimbusJwtDecoder.withJwkSetUri(jwkEndpointURI).build()); - authProvider.setJwtAuthenticationConverter(new JwtAuthenticationConverter()); - } - - /** - * Authenticate a request based on its Spring Security Authentication object - * - * @param authentication Authentication object which contains a JWT to validate - * @return Returns the same authentication object after authentication - */ - @Override - public Authentication authenticate(Authentication authentication) throws AuthenticationException { - return authProvider.authenticate(authentication); - } - - @Override - public boolean supports(Class aClass) { - return authProvider.supports(aClass); - } -} diff --git a/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java b/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java deleted file mode 100644 index e4e398883ec..00000000000 --- a/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import org.springframework.security.core.Authentication; - -/** - * AuthorizationProvider is the base interface that each AuthorizationProvider needs to implement in - * order to authorize requests to Feast Core - */ -public interface AuthorizationProvider { - - /** - * Validates whether a user is allowed access to a project - * - * @param projectId Id of the Feast project - * @param authentication Spring Security Authentication object - * @return AuthorizationResult result of authorization query - */ - AuthorizationResult checkAccessToProject(String projectId, Authentication authentication); -} diff --git a/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java b/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java deleted file mode 100644 index 897cef6d37b..00000000000 --- a/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import java.util.Optional; -import javax.annotation.Nullable; -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * Implementation of AuthorizationProvider will return AuthorizationResult after validating incoming - * requests to Feast Core. AuthorizationResult provides methods to check if user is authorized to - * perform an action or not. - */ -@Getter -@AllArgsConstructor -public class AuthorizationResult { - - /** - * Method to create AuthorizationResult Object. - * - * @param allowed True If user is authorized, False otherwise. - * @param failureReason Reason for authorization failure, if any - * @return AuthorizationResult Object. - */ - public static AuthorizationResult create( - @Nullable boolean allowed, @Nullable String failureReason) { - return new AuthorizationResult(allowed, Optional.ofNullable(failureReason)); - } - - /** - * Method to create failed AuthorizationResult Object. - * - * @param failureReason Reason for authorization failure, if any or Null - * @return AuthorizationResult Object. - */ - public static AuthorizationResult failed(@Nullable String failureReason) { - return new AuthorizationResult(false, Optional.ofNullable(failureReason)); - } - - /** - * Method to create Success AuthorizationResult Object. - * - * @return AuthorizationResult Object. - */ - public static AuthorizationResult success() { - return new AuthorizationResult(true, Optional.empty()); - } - - private boolean allowed; - private Optional failureReason; -} diff --git a/common/src/main/java/feast/common/auth/config/CacheConfiguration.java b/common/src/main/java/feast/common/auth/config/CacheConfiguration.java deleted file mode 100644 index 7731906b88d..00000000000 --- a/common/src/main/java/feast/common/auth/config/CacheConfiguration.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import com.google.common.cache.CacheBuilder; -import feast.common.auth.utils.AuthUtils; -import java.lang.reflect.Method; -import java.util.concurrent.TimeUnit; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.CachingConfigurer; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.concurrent.ConcurrentMapCache; -import org.springframework.cache.concurrent.ConcurrentMapCacheManager; -import org.springframework.cache.interceptor.CacheErrorHandler; -import org.springframework.cache.interceptor.CacheResolver; -import org.springframework.cache.interceptor.KeyGenerator; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.core.Authentication; - -/** CacheConfiguration class defines Cache settings for HttpAuthorizationProvider class. */ -@Configuration -@EnableCaching -@Setter -@Getter -public class CacheConfiguration implements CachingConfigurer { - - private static final int CACHE_SIZE = 10000; - - public static int TTL = 60; - - public static final String AUTHORIZATION_CACHE = "authorization"; - - @Autowired SecurityProperties securityProperties; - - @Bean - public CacheManager cacheManager() { - ConcurrentMapCacheManager cacheManager = - new ConcurrentMapCacheManager(AUTHORIZATION_CACHE) { - - @Override - protected Cache createConcurrentMapCache(final String name) { - return new ConcurrentMapCache( - name, - CacheBuilder.newBuilder() - .expireAfterWrite(TTL, TimeUnit.SECONDS) - .maximumSize(CACHE_SIZE) - .build() - .asMap(), - false); - } - }; - - return cacheManager; - } - - /* - * KeyGenerator used by {@link Cacheable} for caching authorization requests. - * Key format : checkAccessToProject-- - */ - @Bean - public KeyGenerator authKeyGenerator() { - return (Object target, Method method, Object... params) -> { - String projectId = (String) params[0]; - Authentication authentication = (Authentication) params[1]; - String subject = - AuthUtils.getSubjectFromAuth( - authentication, - securityProperties.getAuthorization().getOptions().get("subjectClaim")); - return String.format("%s-%s-%s", method.getName(), projectId, subject); - }; - } - - @Override - public CacheResolver cacheResolver() { - // TODO Auto-generated method stub - return null; - } - - @Override - public KeyGenerator keyGenerator() { - return null; - } - - @Override - public CacheErrorHandler errorHandler() { - // TODO Auto-generated method stub - return null; - } -} diff --git a/common/src/main/java/feast/common/auth/config/SecurityConfig.java b/common/src/main/java/feast/common/auth/config/SecurityConfig.java deleted file mode 100644 index aa7f8a2b353..00000000000 --- a/common/src/main/java/feast/common/auth/config/SecurityConfig.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import feast.common.auth.authentication.DefaultJwtAuthenticationProvider; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.providers.http.HttpAuthorizationProvider; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import net.devh.boot.grpc.server.security.authentication.BearerAuthenticationReader; -import net.devh.boot.grpc.server.security.authentication.GrpcAuthenticationReader; -import net.devh.boot.grpc.server.security.check.AccessPredicateVoter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.access.AccessDecisionManager; -import org.springframework.security.access.AccessDecisionVoter; -import org.springframework.security.access.vote.UnanimousBased; -import org.springframework.security.authentication.AuthenticationManager; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.authentication.ProviderManager; -import org.springframework.security.oauth2.server.resource.BearerTokenAuthenticationToken; - -@Configuration -public class SecurityConfig { - - private final SecurityProperties securityProperties; - - public SecurityConfig(SecurityProperties securityProperties) { - this.securityProperties = securityProperties; - } - - /** - * Initializes an AuthenticationManager if authentication has been enabled. - * - * @return AuthenticationManager - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - AuthenticationManager authenticationManager() { - final List providers = new ArrayList<>(); - - if (securityProperties.getAuthentication().isEnabled()) { - switch (securityProperties.getAuthentication().getProvider()) { - case "jwt": - providers.add( - new DefaultJwtAuthenticationProvider( - securityProperties.getAuthentication().getOptions())); - break; - default: - throw new IllegalArgumentException( - "Please configure an Authentication Provider if you have enabled authentication."); - } - } - return new ProviderManager(providers); - } - - /** - * Creates an AuthenticationReader that the AuthenticationManager will use to authenticate - * requests - * - * @return GrpcAuthenticationReader - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcAuthenticationReader authenticationReader() { - return new BearerAuthenticationReader(BearerTokenAuthenticationToken::new); - } - - /** - * Creates an AccessDecisionManager if authorization is enabled. This object determines the policy - * used to make authorization decisions. - * - * @return AccessDecisionManager - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authorization", name = "enabled") - AccessDecisionManager accessDecisionManager() { - final List> voters = new ArrayList<>(); - voters.add(new AccessPredicateVoter()); - return new UnanimousBased(voters); - } - - /** - * Creates an AuthorizationProvider based on Feast configuration. This provider is available - * through the security service. - * - * @return AuthorizationProvider used to validate access to Feast resources. - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authorization", name = "enabled") - AuthorizationProvider authorizationProvider() { - if (securityProperties.getAuthentication().isEnabled() - && securityProperties.getAuthorization().isEnabled()) { - switch (securityProperties.getAuthorization().getProvider()) { - case "http": - // Merge authenticatoin and authorization options to create HttpAuthorizationProvider. - Map options = securityProperties.getAuthorization().getOptions(); - options.putAll(securityProperties.getAuthentication().getOptions()); - return new HttpAuthorizationProvider(options); - default: - throw new IllegalArgumentException( - "Please configure an Authorization Provider if you have enabled authorization."); - } - } - return null; - } -} diff --git a/common/src/main/java/feast/common/auth/config/SecurityProperties.java b/common/src/main/java/feast/common/auth/config/SecurityProperties.java deleted file mode 100644 index 135cc4b5ed3..00000000000 --- a/common/src/main/java/feast/common/auth/config/SecurityProperties.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import feast.common.validators.OneOfStrings; -import java.util.Map; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class SecurityProperties { - private AuthenticationProperties authentication; - private AuthorizationProperties authorization; - - // Bypass Authentication and Authorization at all HTTP endpoints at /api/v1 - private boolean disableRestControllerAuth; - - @Getter - @Setter - public static class AuthenticationProperties { - // Enable authentication - private boolean enabled; - - // Named authentication provider to use - @OneOfStrings({"jwt"}) - private String provider; - - // K/V options to initialize the provider with - private Map options; - // Key for Subject Claim option which sets the name of the subject claim field in tokens. - public static final String SUBJECT_CLAIM = "subjectClaim"; - } - - @Getter - @Setter - public static class AuthorizationProperties { - // Enable authorization. Authentication must be enabled if authorization is enabled. - private boolean enabled; - - // Named authorization provider to use. - @OneOfStrings({"none", "http"}) - private String provider; - - // K/V options to initialize the provider with - private Map options; - } -} diff --git a/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java b/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java deleted file mode 100644 index aa317cf8b15..00000000000 --- a/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import feast.common.validators.OneOfStrings; -import java.util.Map; - -public class CoreAuthenticationProperties { - // needs to be set to true if authentication is enabled on core - private boolean enabled; - - // authentication provider to use - @OneOfStrings({"google", "oauth"}) - private String provider; - - // K/V options to initialize the provider. - Map options; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getProvider() { - return provider; - } - - public void setProvider(String provider) { - this.provider = provider; - } - - public Map getOptions() { - return options; - } - - public void setOptions(Map options) { - this.options = options; - } -} diff --git a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java b/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java deleted file mode 100644 index 0f42325c5ac..00000000000 --- a/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; - -import com.google.auth.oauth2.IdTokenCredentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import io.grpc.Status; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.Executor; - -/** - * GoogleAuthCredentials provides a Google OIDC ID token for making authenticated gRPC calls. Uses - * Google Application - * Default credentials to obtain the OIDC token used for authentication. The given token will be - * passed as authorization bearer token when making calls. - */ -public class GoogleAuthCredentials extends CallCredentials { - private final IdTokenCredentials credentials; - private static final String BEARER_TYPE = "Bearer"; - private static final Metadata.Key AUTHORIZATION_METADATA_KEY = - Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); - - /** - * Construct a new GoogleAuthCredentials with given options. - * - * @param options a map of options, Required unless specified: audience - Optional, Sets the - * target audience of the token obtained. - */ - public GoogleAuthCredentials(Map options) throws IOException { - String targetAudience = options.getOrDefault("audience", "https://localhost"); - ServiceAccountCredentials serviceCreds = - (ServiceAccountCredentials) - ServiceAccountCredentials.getApplicationDefault() - .createScoped(Arrays.asList("openid", "email")); - - credentials = - IdTokenCredentials.newBuilder() - .setIdTokenProvider(serviceCreds) - .setTargetAudience(targetAudience) - .build(); - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - appExecutor.execute( - () -> { - try { - credentials.refreshIfExpired(); - Metadata headers = new Metadata(); - headers.put( - AUTHORIZATION_METADATA_KEY, - String.format("%s %s", BEARER_TYPE, credentials.getIdToken().getTokenValue())); - applier.apply(headers); - } catch (Throwable e) { - applier.fail(Status.UNAUTHENTICATED.withCause(e)); - } - }); - } - - @Override - public void thisUsesUnstableApi() { - // TODO Auto-generated method stub - - } -} diff --git a/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java b/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java deleted file mode 100644 index f09f4ec9404..00000000000 --- a/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import java.util.concurrent.Executor; - -/** - * JWTCallCredentials provides/attaches a static JWT token for making authenticated gRPC calls. The - * given token will be passed as authorization bearer token when making calls. - */ -public final class JwtCallCredentials extends CallCredentials { - - private String jwt; - - public JwtCallCredentials(String jwt) { - this.jwt = jwt; - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) { - Metadata metadata = new Metadata(); - metadata.put( - Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER), - String.format("Bearer %s", jwt)); - metadataApplier.apply(metadata); - } - - @Override - public void thisUsesUnstableApi() { - // does nothing - } -} diff --git a/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java b/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java deleted file mode 100644 index 429fd7fdadd..00000000000 --- a/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; - -import com.nimbusds.jose.util.JSONObjectUtils; -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import io.grpc.Status; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.Executor; -import javax.security.sasl.AuthenticationException; -import net.minidev.json.JSONObject; -import okhttp3.FormBody; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.springframework.security.oauth2.jwt.NimbusJwtDecoder; - -/** - * OAuthCredentials uses a OAuth OIDC ID token making authenticated gRPC calls. Makes an OAuth - * request to obtain the OIDC token used for authentication. The given token will be passed as - * authorization bearer token when making calls. - */ -public class OAuthCredentials extends CallCredentials { - - private static final String JWK_ENDPOINT_URI = "jwkEndpointURI"; - static final String APPLICATION_JSON = "application/json"; - static final String CONTENT_TYPE = "content-type"; - static final String BEARER_TYPE = "Bearer"; - static final String GRANT_TYPE = "grant_type"; - static final String CLIENT_ID = "client_id"; - static final String CLIENT_SECRET = "client_secret"; - static final String AUDIENCE = "audience"; - static final String OAUTH_URL = "oauth_url"; - static final Metadata.Key AUTHORIZATION_METADATA_KEY = - Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); - - private OkHttpClient httpClient; - private Request request; - private String accessToken; - private Instant tokenExpiryTime; - private NimbusJwtDecoder jwtDecoder; - - /** - * Constructs a new OAuthCredentials with given options. - * - * @param options a map of options, Required unless specified: grant_type - OAuth grant type. - * Should be set as client_credentials audience - Sets the target audience of the token - * obtained. client_id - Client id to use in the OAuth request. client_secret - Client securet - * to use in the OAuth request. jwtEndpointURI - HTTPS URL used to retrieve a JWK that can be - * used to decode the credential. - */ - public OAuthCredentials(Map options) { - this.httpClient = new OkHttpClient(); - if (!(options.containsKey(GRANT_TYPE) - && options.containsKey(CLIENT_ID) - && options.containsKey(AUDIENCE) - && options.containsKey(CLIENT_SECRET) - && options.containsKey(OAUTH_URL) - && options.containsKey(JWK_ENDPOINT_URI))) { - throw new AssertionError( - "please configure the properties:" - + " grant_type, client_id, client_secret, audience, oauth_url, jwkEndpointURI"); - } - RequestBody requestBody = - new FormBody.Builder() - .add(GRANT_TYPE, options.get(GRANT_TYPE)) - .add(CLIENT_ID, options.get(CLIENT_ID)) - .add(CLIENT_SECRET, options.get(CLIENT_SECRET)) - .add(AUDIENCE, options.get(AUDIENCE)) - .build(); - this.request = - new Request.Builder() - .url(options.get(OAUTH_URL)) - .addHeader(CONTENT_TYPE, APPLICATION_JSON) - .post(requestBody) - .build(); - this.jwtDecoder = NimbusJwtDecoder.withJwkSetUri(options.get(JWK_ENDPOINT_URI)).build(); - } - - @Override - public void thisUsesUnstableApi() { - // TODO Auto-generated method stub - - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - appExecutor.execute( - () -> { - try { - // Fetches new token if it is not available or if token has expired. - if (this.accessToken == null || Instant.now().isAfter(this.tokenExpiryTime)) { - Response response = httpClient.newCall(request).execute(); - if (!response.isSuccessful()) { - throw new AuthenticationException(response.message()); - } - JSONObject json = JSONObjectUtils.parse(response.body().string()); - this.accessToken = json.getAsString("access_token"); - this.tokenExpiryTime = jwtDecoder.decode(this.accessToken).getExpiresAt(); - } - Metadata headers = new Metadata(); - headers.put( - AUTHORIZATION_METADATA_KEY, String.format("%s %s", BEARER_TYPE, this.accessToken)); - applier.apply(headers); - } catch (Throwable e) { - applier.fail(Status.UNAUTHENTICATED.withCause(e)); - } - }); - } -} diff --git a/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java b/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java deleted file mode 100644 index 041a6b8e1b0..00000000000 --- a/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.providers.http; - -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.CacheConfiguration; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.providers.http.client.api.DefaultApi; -import feast.common.auth.providers.http.client.invoker.ApiClient; -import feast.common.auth.providers.http.client.invoker.ApiException; -import feast.common.auth.providers.http.client.model.CheckAccessRequest; -import feast.common.auth.utils.AuthUtils; -import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; - -/** - * HTTPAuthorizationProvider uses an external HTTP service for authorizing requests. Please see - * auth/src/main/resources/api.yaml for the API specification of this external service. - */ -public class HttpAuthorizationProvider implements AuthorizationProvider { - - private static final Logger log = LoggerFactory.getLogger(HttpAuthorizationProvider.class); - - private final DefaultApi defaultApiClient; - - /** - * The default subject claim is the key within the Authentication object where the user's identity - * can be found - */ - private final String subjectClaim; - - /** - * Initializes the HTTPAuthorizationProvider - * - * @param options String K/V pair of options to initialize the provider with. Expects at least a - * "basePath" for the provider URL - */ - public HttpAuthorizationProvider(Map options) { - if (options == null) { - throw new IllegalArgumentException( - "Cannot pass empty or null options to HTTPAuthorizationProvider"); - } - - ApiClient apiClient = new ApiClient(); - apiClient.setBasePath(options.get("authorizationUrl")); - this.defaultApiClient = new DefaultApi(apiClient); - subjectClaim = options.get(AuthenticationProperties.SUBJECT_CLAIM); - } - - /** - * Validates whether a user has access to a project. @Cacheable is using {@link - * CacheConfiguration} settings to cache output of the method {@link AuthorizationResult} for a - * specified duration set in cache settings. - * - * @param projectId Name of the Feast project - * @param authentication Spring Security Authentication object - * @return AuthorizationResult result of authorization query - */ - @Cacheable(value = CacheConfiguration.AUTHORIZATION_CACHE, keyGenerator = "authKeyGenerator") - public AuthorizationResult checkAccessToProject(String projectId, Authentication authentication) { - - CheckAccessRequest checkAccessRequest = new CheckAccessRequest(); - Object context = getContext(authentication); - String subject = AuthUtils.getSubjectFromAuth(authentication, subjectClaim); - String resource = "projects:" + projectId; - checkAccessRequest.setAction("ALL"); - checkAccessRequest.setContext(context); - checkAccessRequest.setResource(resource); - checkAccessRequest.setSubject(subject); - try { - Jwt credentials = ((Jwt) authentication.getCredentials()); - // Make authorization request to external service - feast.common.auth.providers.http.client.model.AuthorizationResult authResult = - this.defaultApiClient.checkAccessPost( - checkAccessRequest, "Bearer " + credentials.getTokenValue()); - if (authResult == null) { - throw new RuntimeException( - String.format( - "Empty response returned for access to project %s for subject %s", - projectId, subject)); - } - if (authResult.getAllowed()) { - // Successfully authenticated - return AuthorizationResult.success(); - } - } catch (ApiException e) { - log.error("API exception has occurred during authorization: {}", e.getMessage(), e); - } - - // Could not determine project membership, deny access. - return AuthorizationResult.failed( - String.format("Access denied to project %s for subject %s", projectId, subject)); - } - - /** - * Extract a context object to send as metadata to the authorization service - * - * @param authentication Spring Security Authentication object - * @return Returns a context object that will be serialized and sent as metadata to the - * authorization service - */ - private Object getContext(Authentication authentication) { - // Not implemented yet, left empty - return new Object(); - } -} diff --git a/common/src/main/java/feast/common/auth/service/AuthorizationService.java b/common/src/main/java/feast/common/auth/service/AuthorizationService.java deleted file mode 100644 index 7d325e880d9..00000000000 --- a/common/src/main/java/feast/common/auth/service/AuthorizationService.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.service; - -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import lombok.AllArgsConstructor; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.stereotype.Service; - -@AllArgsConstructor -@Service -public class AuthorizationService { - - private final SecurityProperties securityProperties; - private final AuthorizationProvider authorizationProvider; - - @Autowired - public AuthorizationService( - SecurityProperties securityProperties, - ObjectProvider authorizationProvider) { - this.securityProperties = securityProperties; - this.authorizationProvider = authorizationProvider.getIfAvailable(); - } - - /** - * Determine whether a user has access to a project. - * - * @param securityContext Spring Security Context used to identify a user or service. - * @param project Name of the project for which membership should be tested. - */ - public void authorizeRequest(SecurityContext securityContext, String project) { - Authentication authentication = securityContext.getAuthentication(); - if (!this.securityProperties.getAuthorization().isEnabled()) { - return; - } - - AuthorizationResult result = - this.authorizationProvider.checkAccessToProject(project, authentication); - if (!result.isAllowed()) { - throw new AccessDeniedException(result.getFailureReason().orElse("Access Denied")); - } - } -} diff --git a/common/src/main/java/feast/common/auth/utils/AuthUtils.java b/common/src/main/java/feast/common/auth/utils/AuthUtils.java deleted file mode 100644 index e05fc70a184..00000000000 --- a/common/src/main/java/feast/common/auth/utils/AuthUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.utils; - -import java.util.Map; -import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; - -public class AuthUtils { - // Suppresses default constructor, ensuring non-instantiability. - private AuthUtils() {} - - /** - * Get user email from their authentication object. - * - * @param authentication Spring Security Authentication object, used to extract user details - * @param subjectClaim Indicates the claim where the subject can be found - * @return String user email - */ - public static String getSubjectFromAuth(Authentication authentication, String subjectClaim) { - Jwt principle = ((Jwt) authentication.getPrincipal()); - Map claims = principle.getClaims(); - String subjectValue = (String) claims.getOrDefault(subjectClaim, ""); - - if (subjectValue.isEmpty()) { - throw new IllegalStateException( - String.format("JWT does not have a valid claim %s.", subjectClaim)); - } - - if (subjectClaim.equals("email")) { - boolean validEmail = (new EmailValidator()).isValid(subjectValue, null); - if (!validEmail) { - throw new IllegalStateException("JWT contains an invalid email address"); - } - } - return subjectValue; - } -} diff --git a/common/src/main/java/feast/common/logging/AuditLogger.java b/common/src/main/java/feast/common/logging/AuditLogger.java deleted file mode 100644 index 0b9901e1ade..00000000000 --- a/common/src/main/java/feast/common/logging/AuditLogger.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.logging.config.LoggingProperties; -import feast.common.logging.config.LoggingProperties.AuditLogProperties; -import feast.common.logging.entry.*; -import feast.common.logging.entry.LogResource.ResourceType; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.fluentd.logger.FluentLogger; -import org.slf4j.Marker; -import org.slf4j.MarkerFactory; -import org.slf4j.event.Level; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.info.BuildProperties; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -public class AuditLogger { - private static final String FLUENTD_DESTINATION = "fluentd"; - private static final Marker AUDIT_MARKER = MarkerFactory.getMarker("AUDIT_MARK"); - private static FluentLogger fluentLogger; - private static AuditLogProperties properties; - private static BuildProperties buildProperties; - - @Autowired - public AuditLogger(LoggingProperties loggingProperties, BuildProperties buildProperties) { - // Spring runs this constructor when creating the AuditLogger bean, - // which allows us to populate the AuditLogger class with dependencies. - // This allows us to use the dependencies in the AuditLogger's static methods - AuditLogger.properties = loggingProperties.getAudit(); - AuditLogger.buildProperties = buildProperties; - if (AuditLogger.properties.getMessageLogging() != null - && AuditLogger.properties.getMessageLogging().isEnabled()) { - AuditLogger.fluentLogger = - FluentLogger.getLogger( - "feast", - AuditLogger.properties.getMessageLogging().getFluentdHost(), - AuditLogger.properties.getMessageLogging().getFluentdPort()); - } - } - - /** - * Log the handling of a Protobuf message by a service call. - * - * @param entryBuilder with all fields set except instance. - */ - public static void logMessage(Level level, MessageAuditLogEntry.Builder entryBuilder) { - log( - level, - entryBuilder - .setComponent(buildProperties.getArtifact()) - .setVersion(buildProperties.getVersion()) - .build()); - } - - /** - * Log an action being taken on a specific resource - * - * @param level describing the severity of the log. - * @param action name of the action being taken on specific resource. - * @param resourceType the type of resource being logged. - * @param resourceId resource specific identifier identifing the instance of the resource. - */ - public static void logAction( - Level level, String action, ResourceType resourceType, String resourceId) { - log( - level, - ActionAuditLogEntry.of( - buildProperties.getArtifact(), - buildProperties.getArtifact(), - LogResource.of(resourceType, resourceId), - action)); - } - - /** - * Log a transition in state/status in a specific resource. - * - * @param level describing the severity of the log. - * @param status name of end status which the resource transition to. - * @param resourceType the type of resource being logged. - * @param resourceId resource specific identifier identifing the instance of the resource. - */ - public static void logTransition( - Level level, String status, ResourceType resourceType, String resourceId) { - log( - level, - TransitionAuditLogEntry.of( - buildProperties.getArtifact(), - buildProperties.getArtifact(), - LogResource.of(resourceType, resourceId), - status)); - } - - /** - * Log given {@link AuditLogEntry} at the given logging {@link Level} to the Audit log. - * - * @param level describing the severity of the log. - * @param entry the {@link AuditLogEntry} to push to the audit log. - */ - private static void log(Level level, AuditLogEntry entry) { - // Check if audit logging is of this specific log entry enabled. - if (!properties.isEnabled()) { - return; - } - - // Either forward log to logging layer or log to console - String destination = properties.getMessageLogging().getDestination(); - if (destination.equals(FLUENTD_DESTINATION)) { - if (entry.getKind() == AuditLogEntryKind.MESSAGE) { - Map fluentdLogs = new HashMap<>(); - MessageAuditLogEntry messageAuditLogEntry = (MessageAuditLogEntry) entry; - String releaseName; - - try { - releaseName = - StringUtils.defaultIfEmpty( - System.getenv("RELEASE_NAME"), InetAddress.getLocalHost().getHostAddress()); - } catch (UnknownHostException e) { - releaseName = StringUtils.defaultIfEmpty(System.getenv("RELEASE_NAME"), ""); - } - - fluentdLogs.put("id", messageAuditLogEntry.getId()); - fluentdLogs.put("identity", messageAuditLogEntry.getIdentity()); - fluentdLogs.put("service", messageAuditLogEntry.getService()); - fluentdLogs.put("status_code", messageAuditLogEntry.getStatusCode()); - fluentdLogs.put("method", messageAuditLogEntry.getMethod()); - fluentdLogs.put("release_name", releaseName); - try { - fluentdLogs.put("request", JsonFormat.printer().print(messageAuditLogEntry.getRequest())); - fluentdLogs.put( - "response", JsonFormat.printer().print(messageAuditLogEntry.getResponse())); - } catch (InvalidProtocolBufferException e) { - } - fluentLogger.log("fluentd", fluentdLogs); - } - } else { - // Log event to audit log through enabled formats - String entryJSON = entry.toJSON(); - switch (level) { - case TRACE: - log.trace(AUDIT_MARKER, entryJSON); - break; - case DEBUG: - log.debug(AUDIT_MARKER, entryJSON); - break; - case INFO: - log.info(AUDIT_MARKER, entryJSON); - break; - case WARN: - log.warn(AUDIT_MARKER, entryJSON); - break; - case ERROR: - log.error(AUDIT_MARKER, entryJSON); - break; - } - } - } -} diff --git a/common/src/main/java/feast/common/logging/config/LoggingProperties.java b/common/src/main/java/feast/common/logging/config/LoggingProperties.java deleted file mode 100644 index 06e62f71af3..00000000000 --- a/common/src/main/java/feast/common/logging/config/LoggingProperties.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.config; - -import feast.common.validators.OneOfStrings; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class LoggingProperties { - @NotNull private AuditLogProperties audit; - - @Getter - @Setter - public static class AuditLogProperties { - // Whether to enable/disable audit logging entirely. - private boolean enabled; - - private MessageLogging messageLogging; - - @Getter - @Setter - public static class MessageLogging { - // Whether to enable/disable message level (ie request/response) audit logging. - private boolean enabled; - - // Whether to log to console or fluentd - @OneOfStrings({"console", "fluentd"}) - private String destination; - - // fluentD service host for external (request/response) logging. - private String fluentdHost; - - // fluentD service port for external (request/response) logging. - private Integer fluentdPort; - } - } -} diff --git a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java deleted file mode 100644 index cec85b736a6..00000000000 --- a/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import com.google.auto.value.AutoValue; - -/** ActionAuditLogEntry records an action being taken on a specific resource */ -@AutoValue -public abstract class ActionAuditLogEntry extends AuditLogEntry { - /** The name of the action taken on the resource. */ - public abstract String getAction(); - - /** The target resource of which the action was taken on. */ - public abstract LogResource getResource(); - - /** - * Create an {@link AuditLogEntry} that records an action being taken on a specific resource. - * - * @param component The name of th Feast component producing this {@link AuditLogEntry}. - * @param version The version of Feast producing this {@link AuditLogEntry}. - * @param resource The target resource of which the action was taken on. - * @param action The name of the action being taken on the given resource. - */ - public static ActionAuditLogEntry of( - String component, String version, LogResource resource, String action) { - return new AutoValue_ActionAuditLogEntry( - component, version, AuditLogEntryKind.ACTION, action, resource); - } -} diff --git a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java deleted file mode 100644 index 9aa8fcb8c5c..00000000000 --- a/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import com.google.gson.Gson; - -/** - * AuditLogEntry represents a single audit Log Entry. Audit log entry can converted into string with - * {{@link #toString()} for human readable representation. Or structured JSON with {{@link - * #toJSON()} for a machine parsable representation. - */ -public abstract class AuditLogEntry { - /** Declare Log Type to allow external Logging systems to filter out {@link AuditLogEntry} */ - public final String logType = "FeastAuditLogEntry"; - - public final String application = "Feast"; - - /** The name of the Feast component producing this {@link AuditLogEntry} */ - public abstract String getComponent(); - - /** The version of Feast producing this {@link AuditLogEntry} */ - public abstract String getVersion(); - - public abstract AuditLogEntryKind getKind(); - - /** Return a structured JSON representation of this {@link AuditLogEntry} */ - public String toJSON() { - Gson gson = new Gson(); - return gson.toJson(this); - } -} diff --git a/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java b/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java deleted file mode 100644 index d673f6bdb30..00000000000 --- a/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -/** AuditLogEntryKind lists the various kinds of {@link AuditLogEntry} */ -public enum AuditLogEntryKind { - MESSAGE, - ACTION, - TRANSITION, -} diff --git a/common/src/main/java/feast/common/logging/entry/LogResource.java b/common/src/main/java/feast/common/logging/entry/LogResource.java deleted file mode 100644 index 1d0345a4042..00000000000 --- a/common/src/main/java/feast/common/logging/entry/LogResource.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import com.google.auto.value.AutoValue; - -@AutoValue -/** - * LogResource is used in {@link AuditLogEntry} to reference a specific resource as the subject of - * the log - */ -public abstract class LogResource { - public enum ResourceType { - JOB, - FEATURE_TABLE - } - - public abstract ResourceType getType(); - - public abstract String getId(); - - public static LogResource of(ResourceType type, String id) { - return new AutoValue_LogResource(type, id); - } -} diff --git a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java deleted file mode 100644 index 745cc1283ae..00000000000 --- a/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import com.google.auto.value.AutoValue; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; -import com.google.protobuf.Empty; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.util.JsonFormat; -import io.grpc.Status.Code; -import java.lang.reflect.Type; -import java.util.UUID; - -/** MessageAuditLogEntry records the handling of a Protobuf message by a service call. */ -@AutoValue -public abstract class MessageAuditLogEntry extends AuditLogEntry { - /** Id used to identify the service call that the log entry is recording */ - public abstract UUID getId(); - - /** The name of the service that was used to handle the service call. */ - public abstract String getService(); - - /** The name of the method that was used to handle the service call. */ - public abstract String getMethod(); - - /** The request Protobuf {@link Message} that was passed to the Service in the service call. */ - public abstract Message getRequest(); - - /** - * The response Protobuf {@link Message} that was passed to the Service in the service call. May - * be an {@link Empty} protobuf no request could be collected due to an error. - */ - public abstract Message getResponse(); - - /** - * The authenticated identity that was assumed during the handling of the service call. For - * example, the user id or email that identifies the user making the call. Empty if the service - * call is not authenticated. - */ - public abstract String getIdentity(); - - /** The result status code of the service call. */ - public abstract Code getStatusCode(); - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setId(UUID id); - - public abstract Builder setComponent(String component); - - public abstract Builder setVersion(String component); - - public abstract Builder setKind(AuditLogEntryKind kind); - - public abstract Builder setService(String name); - - public abstract Builder setMethod(String name); - - public abstract Builder setRequest(Message request); - - public abstract Builder setResponse(Message response); - - public abstract Builder setIdentity(String identity); - - public abstract Builder setStatusCode(Code statusCode); - - public abstract MessageAuditLogEntry build(); - } - - public static MessageAuditLogEntry.Builder newBuilder() { - return new AutoValue_MessageAuditLogEntry.Builder() - .setKind(AuditLogEntryKind.MESSAGE) - .setId(UUID.randomUUID()); - } - - @Override - public String toJSON() { - // GSON requires custom typeadapter (serializer) to convert Protobuf messages to JSON properly - Gson gson = - new GsonBuilder() - .registerTypeAdapter( - Message.class, - new JsonSerializer() { - @Override - public JsonElement serialize( - Message message, Type type, JsonSerializationContext context) { - try { - String messageJSON = JsonFormat.printer().print(message); - return new JsonParser().parse(messageJSON); - } catch (InvalidProtocolBufferException e) { - - throw new RuntimeException( - "Unexpected exception converting Protobuf to JSON", e); - } - } - }) - .create(); - return gson.toJson(this); - } -} diff --git a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java b/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java deleted file mode 100644 index 0f139b7bdbd..00000000000 --- a/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import com.google.auto.value.AutoValue; - -/** TransitionAuditLogEntry records a transition in state/status in a specific resource. */ -@AutoValue -public abstract class TransitionAuditLogEntry extends AuditLogEntry { - /** The resource which the state/status transition occured. */ - public abstract LogResource getResource(); - - /** The end status with the resource transition to. */ - public abstract String getStatus(); - - /** - * Construct a new {@link AuditLogEntry} to record a transition in state/status in a specific - * resource. - * - * @param component The name of th Feast component producing this {@link AuditLogEntry}. - * @param version The version of Feast producing this {@link AuditLogEntry}. - * @param resource the resource which the transtion occured - * @param status the end status which the resource transitioned to. - */ - public static TransitionAuditLogEntry of( - String component, String version, LogResource resource, String status) { - return new AutoValue_TransitionAuditLogEntry( - component, version, AuditLogEntryKind.TRANSITION, resource, status); - } -} diff --git a/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java b/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java deleted file mode 100644 index da895f0093b..00000000000 --- a/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.interceptors; - -import com.google.protobuf.Empty; -import com.google.protobuf.Message; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.utils.AuthUtils; -import feast.common.logging.AuditLogger; -import feast.common.logging.config.LoggingProperties; -import feast.common.logging.entry.MessageAuditLogEntry; -import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; -import io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCall.Listener; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; -import java.util.Map; -import org.slf4j.event.Level; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.lang.Nullable; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.stereotype.Component; - -/** - * GrpcMessageInterceptor intercepts a GRPC calls to log handling of GRPC messages to the Audit Log. - * Intercepts the incoming and outgoing messages logs them to the audit log, together with method - * name and assumed authenticated identity (if authentication is enabled). NOTE: - * GrpcMessageInterceptor assumes that all service calls are unary (ie single request/response). - */ -@Component -public class GrpcMessageInterceptor implements ServerInterceptor { - private SecurityProperties securityProperties; - private LoggingProperties loggingProperties; - - /** - * Construct GrpcMessageIntercetor. - * - * @param loggingProperties properties used to configure logging interceptor. - * @param securityProperties If provided, will output the subject claim specified in - * securityProperties as identity in {@link MessageAuditLogEntry} instead. - */ - @Autowired - public GrpcMessageInterceptor( - LoggingProperties loggingProperties, @Nullable SecurityProperties securityProperties) { - this.securityProperties = securityProperties; - this.loggingProperties = loggingProperties; - } - - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - // Disable the message logging interceptor entirely if message logging is disabled. - if (!loggingProperties.getAudit().getMessageLogging().isEnabled()) { - return next.startCall(call, headers); - } - - MessageAuditLogEntry.Builder entryBuilder = MessageAuditLogEntry.newBuilder(); - // default response/request message to empty proto in log entry. - // request could be empty when the client closes the connection before sending a request - // message. - // response could be unset when the service encounters an error when processsing the service - // call. - entryBuilder.setRequest(Empty.newBuilder().build()); - entryBuilder.setResponse(Empty.newBuilder().build()); - - // Unpack service & method name from call - // full method name is in format ./ - String fullMethodName = call.getMethodDescriptor().getFullMethodName(); - entryBuilder.setService( - fullMethodName.substring(fullMethodName.lastIndexOf(".") + 1, fullMethodName.indexOf("/"))); - entryBuilder.setMethod(fullMethodName.substring(fullMethodName.indexOf("/") + 1)); - - // Attempt Extract current authenticated identity. - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - String identity = (authentication != null) ? getIdentity(authentication) : ""; - entryBuilder.setIdentity(identity); - - // Register forwarding call to intercept outgoing response and log to audit log - call = - new SimpleForwardingServerCall(call) { - @Override - public void sendMessage(RespT message) { - // 2. Track the response & Log entry to audit logger - super.sendMessage(message); - entryBuilder.setResponse((Message) message); - } - - @Override - public void close(Status status, Metadata trailers) { - super.close(status, trailers); - // 3. Log the message log entry to the audit log - Level logLevel = (status.isOk()) ? Level.INFO : Level.ERROR; - entryBuilder.setStatusCode(status.getCode()); - AuditLogger.logMessage(logLevel, entryBuilder); - } - }; - - ServerCall.Listener listener = next.startCall(call, headers); - return new SimpleForwardingServerCallListener(listener) { - @Override - // Register listener to intercept incoming request messages and log to audit log - public void onMessage(ReqT message) { - super.onMessage(message); - // 1. Track the request. - entryBuilder.setRequest((Message) message); - } - }; - } - - /** - * Extract current authenticated identity from given {@link Authentication}. Extracts subject - * claim if specified in AuthorizationProperties, otherwise returns authentication subject. - */ - private String getIdentity(Authentication authentication) { - // use subject claim as identity if set in security authorization properties - if (securityProperties != null) { - Map options = securityProperties.getAuthentication().getOptions(); - if (options.containsKey(AuthenticationProperties.SUBJECT_CLAIM)) { - try { - return AuthUtils.getSubjectFromAuth( - authentication, options.get(AuthenticationProperties.SUBJECT_CLAIM)); - } catch (IllegalStateException e) { - // could not extract claim, revert to authenticated name. - return authentication.getName(); - } - } - } - return authentication.getName(); - } -} diff --git a/common/src/main/java/feast/common/models/FeatureTable.java b/common/src/main/java/feast/common/models/FeatureTable.java deleted file mode 100644 index d4712e7a013..00000000000 --- a/common/src/main/java/feast/common/models/FeatureTable.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; - -public class FeatureTable { - - /** - * Accepts FeatureTableSpec object and returns its reference in String - * "project/featuretable_name". - * - * @param featureTableSpec {@link FeatureTableSpec} - * @return String format of FeatureTableReference - */ - public static String getFeatureTableStringRef(String project, FeatureTableSpec featureTableSpec) { - return String.format("%s/%s", project, featureTableSpec.getName()); - } - - /** - * Accepts FeatureReferenceV2 object and returns its reference in String - * "project/featuretable_name". - * - * @param featureReference {@link FeatureReferenceV2} - * @return String format of FeatureTableReference - */ - public static String getFeatureTableStringRef( - String project, FeatureReferenceV2 featureReference) { - return String.format("%s/%s", project, featureReference.getFeatureTable()); - } -} diff --git a/common/src/main/java/feast/common/models/FeatureV2.java b/common/src/main/java/feast/common/models/FeatureV2.java deleted file mode 100644 index 8debca33b9f..00000000000 --- a/common/src/main/java/feast/common/models/FeatureV2.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; - -public class FeatureV2 { - - /** - * Accepts FeatureReferenceV2 object and returns its reference in String - * "featuretable_name:feature_name". - * - * @param featureReference {@link FeatureReferenceV2} - * @return String format of FeatureReferenceV2 - */ - public static String getFeatureStringRef(FeatureReferenceV2 featureReference) { - String ref = featureReference.getName(); - if (!featureReference.getFeatureTable().isEmpty()) { - ref = featureReference.getFeatureTable() + ":" + ref; - } - return ref; - } -} diff --git a/common/src/main/java/feast/common/models/Store.java b/common/src/main/java/feast/common/models/Store.java deleted file mode 100644 index 1701b0bb3a2..00000000000 --- a/common/src/main/java/feast/common/models/Store.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class Store { - - /** - * Accepts a comma-delimited string and converts it to a list of Subscription class objects. - * - * @param subscriptions String formatted Subscriptions, comma delimited. - * @return List of Subscription class objects - */ - public static List parseSubFromStr(String subscriptions) { - List allSubscriptions = - Arrays.stream(subscriptions.split(",")) - .map(subscriptionStr -> convertStringToSubscription(subscriptionStr)) - .collect(Collectors.toList()); - - return allSubscriptions; - } - - /** - * Accepts a comma-delimited string and converts it to a list of Subscription class objects, with - * exclusions filtered out. - * - * @param subscriptions String formatted Subscriptions, comma delimited. - * @return List of Subscription class objects - */ - public static List parseSubFromStrWithoutExclusions(String subscriptions) { - List allSubscriptions = - Arrays.stream(subscriptions.split(",")) - .map(subscriptionStr -> convertStringToSubscription(subscriptionStr)) - .collect(Collectors.toList()); - - allSubscriptions = - allSubscriptions.stream().filter(sub -> !sub.getExclude()).collect(Collectors.toList()); - - return allSubscriptions; - } - - /** - * Accepts a Subscription class object and returns it in string format - * - * @param subscription Subscription class to be converted to string format - * @return String formatted Subscription class - */ - public static String parseSubscriptionFrom(Subscription subscription) { - if (subscription.getName().isEmpty() || subscription.getProject().isEmpty()) { - throw new IllegalArgumentException( - String.format("Missing arguments in subscription string: %s", subscription.toString())); - } - - return String.format( - "%s:%s:%s", subscription.getProject(), subscription.getName(), subscription.getExclude()); - } - - /** - * Accepts a exclude parameter to determine whether to return subscriptions that are excluded. - * - * @param subscription String formatted Subscription to be converted to Subscription class - * @return Subscription class with its respective attributes - */ - public static Subscription convertStringToSubscription(String subscription) { - if (subscription.equals("")) { - return Subscription.newBuilder().build(); - } - String[] split = subscription.split(":"); - if (split.length == 2) { - // Backward compatibility check - return Subscription.newBuilder().setProject(split[0]).setName(split[1]).build(); - } - return Subscription.newBuilder() - .setProject(split[0]) - .setName(split[1]) - .setExclude(Boolean.parseBoolean(split[2])) - .build(); - } - - /** - * The current use of this function is to determine whether a FeatureRow is subscribed to a - * Featureset. - * - * @param subscriptions List of Subscriptions available in Store - * @param projectName Project name used for matching Subscription's Project - * @param featureSetName Featureset name used for matching Subscription's Featureset - * @return boolean flag to signify if FeatureRow is subscribed to Featureset - */ - public static boolean isSubscribedToFeatureSet( - List subscriptions, String projectName, String featureSetName) { - // Case 1: Highest priority check, to exclude all matching subscriptions with excluded flag = - // true - for (Subscription sub : subscriptions) { - // If configuration missing, fail - if (sub.getProject().isEmpty() || sub.getName().isEmpty()) { - throw new IllegalArgumentException( - String.format("Subscription is missing arguments: %s", sub.toString())); - } - // Match feature set name to pattern - Pattern patternName = getNamePattern(sub); - Pattern patternProject = getProjectPattern(sub); - // SubCase: Project name and feature set name matches and excluded flag is true - if (patternProject.matcher(projectName).matches() - && patternName.matcher(featureSetName).matches() - && sub.getExclude()) { - return false; - } - } - // Case 2: Featureset is not excluded, check if it is included in the current subscriptions - // filteredSubscriptions only contain subscriptions with excluded flag = false - List filteredSubscriptions = - subscriptions.stream().filter(sub -> !sub.getExclude()).collect(Collectors.toList()); - - for (Subscription filteredSub : filteredSubscriptions) { - // Match feature set name to pattern - Pattern patternName = getNamePattern(filteredSub); - Pattern patternProject = getProjectPattern(filteredSub); - // SubCase: Project name and feature set name matches - if (patternProject.matcher(projectName).matches() - && patternName.matcher(featureSetName).matches()) { - return true; - } - } - return false; - } - - private static Pattern getProjectPattern(Subscription subscription) { - String subProject = subscription.getProject(); - if (!subscription.getProject().contains(".*")) { - subProject = subProject.replace("*", ".*"); - } - - return Pattern.compile(subProject); - } - - private static Pattern getNamePattern(Subscription subscription) { - String subName = subscription.getName(); - if (!subscription.getProject().contains(".*")) { - subName = subName.replace("*", ".*"); - } - - return Pattern.compile(subName); - } -} diff --git a/common/src/main/java/feast/common/util/KafkaSerialization.java b/common/src/main/java/feast/common/util/KafkaSerialization.java deleted file mode 100644 index 62c31a63bb7..00000000000 --- a/common/src/main/java/feast/common/util/KafkaSerialization.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.util; - -import com.google.protobuf.GeneratedMessageV3; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.Parser; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; - -/* -Serializer & Deserializer implementation to write & read protobuf object from/to kafka - */ -public class KafkaSerialization { - public static class ProtoSerializer implements Serializer { - @Override - public byte[] serialize(String topic, T data) { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - try { - data.writeTo(stream); - } catch (IOException e) { - throw new RuntimeException( - String.format( - "Unable to serialize object of type %s. Reason: %s", - data.getClass().getName(), e.getCause().getMessage())); - } - - return stream.toByteArray(); - } - } - - public static class ProtoDeserializer implements Deserializer { - private Parser parser; - - public ProtoDeserializer(Parser parser) { - this.parser = parser; - } - - @Override - public T deserialize(String topic, byte[] data) { - try { - return parser.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - String.format( - "Unable to deserialize object from topic %s. Reason: %s", - topic, e.getCause().getMessage())); - } - } - } -} diff --git a/common/src/main/java/feast/common/validators/OneOfStringValidator.java b/common/src/main/java/feast/common/validators/OneOfStringValidator.java deleted file mode 100644 index 42428bd8c05..00000000000 --- a/common/src/main/java/feast/common/validators/OneOfStringValidator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.validators; - -import java.util.Arrays; -import javax.validation.ConstraintValidator; -import javax.validation.ConstraintValidatorContext; - -/** Validates whether a string value is found within a collection. */ -public class OneOfStringValidator implements ConstraintValidator { - - /** Values that are permitted for a specific instance of this validator */ - String[] allowedValues; - - /** - * Initialize the OneOfStringValidator with a collection of allowed String values. - * - * @param constraintAnnotation - */ - @Override - public void initialize(OneOfStrings constraintAnnotation) { - allowedValues = constraintAnnotation.value(); - } - - /** - * Validates whether a string value is found within the collection defined in the annotation. - * - * @param value String value that should be validated - * @param context Provides contextual data and operation when applying a given constraint - * validator - * @return Boolean value indicating whether the string is found within the allowed values. - */ - @Override - public boolean isValid(String value, ConstraintValidatorContext context) { - return Arrays.asList(allowedValues).contains(value); - } -} diff --git a/common/src/main/java/feast/common/validators/OneOfStrings.java b/common/src/main/java/feast/common/validators/OneOfStrings.java deleted file mode 100644 index b0acfae09d9..00000000000 --- a/common/src/main/java/feast/common/validators/OneOfStrings.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.validators; - -import java.lang.annotation.*; -import javax.validation.Constraint; -import javax.validation.Payload; - -/** - * Annotation for String "one of" validation. Allows for the definition of a collection through an - * annotation. The collection is used to test values defined in the object. - */ -@Target({ - ElementType.METHOD, - ElementType.FIELD, - ElementType.ANNOTATION_TYPE, - ElementType.CONSTRUCTOR, - ElementType.PARAMETER -}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Constraint(validatedBy = OneOfStringValidator.class) -public @interface OneOfStrings { - /** @return Default error message that is returned if the incorrect value is set */ - String message() default "Field value must be one of the following: {value}"; - - /** Allows for the specification of validation groups to which this constraint belongs. */ - Class[] groups() default {}; - - /** An attribute payload that can be used to assign custom payload objects to a constraint. */ - Class[] payload() default {}; - - /** @return Default value that is returned if no allowed values are configured */ - String[] value() default {}; -} diff --git a/common/src/main/resources/api.yaml b/common/src/main/resources/api.yaml deleted file mode 100644 index 73ddaf2c2dd..00000000000 --- a/common/src/main/resources/api.yaml +++ /dev/null @@ -1,117 +0,0 @@ -openapi: 3.0.1 -info: - description: 'Feast Authorization Server' - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - title: Feast Authorization Server - version: 1.0.0 -servers: - - url: / -paths: - /healthz: - get: - responses: - "200": - description: Online - "500": - description: Offline - /readiness: - get: - responses: - "200": - description: Ready - "500": - description: Not Ready - /checkAccess: - post: - parameters: - - name: Authorization - in: header - description: Auth token - schema: - type: string - operationId: check_access_post - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/checkAccessRequest' - description: Request containing user, resource, and action information. Used to make an authorization decision. - required: true - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/authorizationResult' - description: Authorization passed response - "403": - content: - application/json: - schema: - $ref: '#/components/schemas/authorizationResult' - description: Authorization failed response - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_500' - description: The standard error format - summary: Check whether request is authorized to access a specific resource - x-codegen-request-body-name: body -components: - schemas: - checkAccessRequest: - example: - action: 'read' - context: '{}' - resource: 'feast:project' - subject: 'me@example.com' - properties: - action: - description: Action is the action that is being taken on the requested resource. - type: string - context: - description: Context is the request's environmental context. - properties: {} - type: object - resource: - description: Resource is the resource that access is requested to. - type: string - subject: - description: Subject is the subject that is requesting access, typically the user. - type: string - title: Input for checking if a request is allowed or not. - type: object - authorizationResult: - example: - allowed: true - properties: - allowed: - description: Allowed is true if the request should be allowed and false - otherwise. - type: boolean - required: - - allowed - title: AuthorizationResult is the result of an access control decision. It contains - the decision outcome. - type: object - inline_response_500: - properties: - code: - format: int64 - type: integer - details: - items: - properties: {} - type: object - type: array - message: - type: string - reason: - type: string - request: - type: string - status: - type: string \ No newline at end of file diff --git a/common/src/main/resources/log4j2.xml b/common/src/main/resources/log4j2.xml deleted file mode 100644 index c75c2db13cc..00000000000 --- a/common/src/main/resources/log4j2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java b/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java deleted file mode 100644 index f303801c301..00000000000 --- a/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import com.google.common.collect.ImmutableMap; -import feast.common.auth.config.CacheConfiguration; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.providers.http.HttpAuthorizationProvider; -import feast.common.auth.providers.http.client.api.DefaultApi; -import feast.common.auth.providers.http.client.model.AuthorizationResult; -import feast.common.auth.providers.http.client.model.CheckAccessRequest; -import java.util.HashMap; -import java.util.Map; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@ContextConfiguration( - classes = {CacheConfiguration.class, HttpAuthorizationProviderCachingTest.Config.class}) -public class HttpAuthorizationProviderCachingTest { - - // static since field needs to updated in provider() bean - private static DefaultApi api = Mockito.mock(DefaultApi.class); - - @Autowired AuthorizationProvider provider; - - @Configuration - static class Config { - @Bean - SecurityProperties securityProps() { - // setting TTL static variable in SecurityProperties bean, since CacheConfiguration bean is - // dependent on SecurityProperties. - CacheConfiguration.TTL = 1; - AuthenticationProperties authentication = Mockito.mock(AuthenticationProperties.class); - AuthorizationProperties authorization = new AuthorizationProperties(); - authorization.setEnabled(true); - authorization.setProvider("http"); - authorization.setOptions(ImmutableMap.of("authorizationUrl", "localhost")); - - authentication.setOptions(ImmutableMap.of("subjectClaim", "email")); - - SecurityProperties sp = new SecurityProperties(); - sp.setAuthentication(authentication); - sp.setAuthorization(authorization); - return sp; - } - - @Bean - AuthorizationProvider provider() throws NoSuchFieldException, SecurityException { - Map options = new HashMap<>(); - options.put("authorizationUrl", "localhost"); - options.put("subjectClaim", "email"); - HttpAuthorizationProvider provider = new HttpAuthorizationProvider(options); - FieldSetter.setField(provider, provider.getClass().getDeclaredField("defaultApiClient"), api); - return provider; - } - } - - @Test - public void testCheckAccessToProjectShouldReadFromCacheWhenAvailable() throws Exception { - Authentication auth = Mockito.mock(Authentication.class); - Jwt jwt = Mockito.mock(Jwt.class); - Map claims = new HashMap<>(); - claims.put("email", "test@test.com"); - doReturn(jwt).when(auth).getCredentials(); - doReturn(jwt).when(auth).getPrincipal(); - doReturn(claims).when(jwt).getClaims(); - doReturn("test_token").when(jwt).getTokenValue(); - AuthorizationResult authResult = new AuthorizationResult(); - authResult.setAllowed(true); - doReturn(authResult) - .when(api) - .checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - - // Should save the result in cache - provider.checkAccessToProject("test", auth); - // Should read from cache - provider.checkAccessToProject("test", auth); - verify(api, times(1)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - - // cache ttl is set to 1 second for testing. - Thread.sleep(1100); - - // Should make an invocation to external service - provider.checkAccessToProject("test", auth); - verify(api, times(2)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - // Should read from cache - provider.checkAccessToProject("test", auth); - verify(api, times(2)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - } -} diff --git a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java b/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java deleted file mode 100644 index cf355e09e4b..00000000000 --- a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.logging.entry; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import feast.common.logging.entry.LogResource.ResourceType; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.ValueProto.Value; -import io.grpc.Status; -import java.util.Arrays; -import java.util.List; -import org.junit.Test; - -public class AuditLogEntryTest { - public List getTestAuditLogs() { - GetOnlineFeaturesRequestV2 requestSpec = - GetOnlineFeaturesRequestV2.newBuilder() - .addAllFeatures( - Arrays.asList( - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature1") - .build(), - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature2") - .build())) - .build(); - - GetOnlineFeaturesResponse responseSpec = - GetOnlineFeaturesResponse.newBuilder() - .addAllFieldValues( - Arrays.asList( - FieldValues.newBuilder() - .putFields( - "featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build()) - .build(), - FieldValues.newBuilder() - .putFields( - "featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build()) - .build())) - .build(); - - return Arrays.asList( - MessageAuditLogEntry.newBuilder() - .setComponent("feast-serving") - .setVersion("0.9") - .setService("ServingService") - .setMethod("getOnlineFeatures") - .setRequest(requestSpec) - .setResponse(responseSpec) - .setStatusCode(Status.OK.getCode()) - .setIdentity("adam@no.such.email") - .build(), - ActionAuditLogEntry.of( - "core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"), - TransitionAuditLogEntry.of( - "core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY")); - } - - @Test - public void shouldReturnJSONRepresentationOfAuditLog() { - for (AuditLogEntry auditLog : getTestAuditLogs()) { - // Check that auditLog's toJSON() returns valid JSON - String logJSON = auditLog.toJSON(); - System.out.println(logJSON); - JsonParser parser = new JsonParser(); - - // check basic fields are present in JSON representation. - JsonObject logObject = parser.parse(logJSON).getAsJsonObject(); - assertThat(logObject.getAsJsonPrimitive("logType").getAsString(), equalTo(auditLog.logType)); - assertThat( - logObject.getAsJsonPrimitive("kind").getAsString(), equalTo(auditLog.getKind().name())); - } - } -} diff --git a/common/src/test/java/feast/common/models/FeaturesTest.java b/common/src/test/java/feast/common/models/FeaturesTest.java deleted file mode 100644 index 180f7e4e697..00000000000 --- a/common/src/test/java/feast/common/models/FeaturesTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import org.junit.Before; -import org.junit.Test; - -public class FeaturesTest { - - private FeatureReferenceV2 featureReference; - - @Before - public void setUp() { - featureReference = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature1") - .build(); - } - - @Test - public void shouldReturnFeatureStringRef() { - String actualFeatureStringRef = FeatureV2.getFeatureStringRef(featureReference); - String expectedFeatureStringRef = "featuretable_1:feature1"; - - assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef)); - } -} diff --git a/common/src/test/java/feast/common/models/StoreTest.java b/common/src/test/java/feast/common/models/StoreTest.java deleted file mode 100644 index 1acba12d2b6..00000000000 --- a/common/src/test/java/feast/common/models/StoreTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertTrue; - -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.Arrays; -import java.util.List; -import org.junit.Before; -import org.junit.Test; - -public class StoreTest { - - private List allSubscriptions; - - @Before - public void setUp() { - - Subscription emptySubscription = Subscription.newBuilder().build(); - Subscription subscription1 = Subscription.newBuilder().setProject("*").setName("*").build(); - Subscription subscription2 = - Subscription.newBuilder().setProject("project1").setName("fs_2").build(); - Subscription subscription3 = - Subscription.newBuilder().setProject("project1").setName("fs_1").setExclude(true).build(); - allSubscriptions = - Arrays.asList(emptySubscription, subscription1, subscription2, subscription3); - } - - @Test - public void shouldReturnSubscriptionsBasedOnStr() { - String subscriptions = "project1:fs_1:true,project1:fs_2"; - List actual1 = Store.parseSubFromStr(subscriptions); - List expected1 = Arrays.asList(allSubscriptions.get(2), allSubscriptions.get(3)); - - List actual2 = Store.parseSubFromStrWithoutExclusions(subscriptions); - List expected2 = Arrays.asList(allSubscriptions.get(2)); - - assertTrue(actual1.containsAll(expected1) && expected1.containsAll(actual1)); - assertTrue(actual2.containsAll(expected2) && expected2.containsAll(actual2)); - } - - @Test - public void shouldReturnStringBasedOnSubscription() { - // Case: default exclude should be false - String actual1 = Store.parseSubscriptionFrom(allSubscriptions.get(2)); - Subscription sub1 = allSubscriptions.get(2); - String expected1 = sub1.getProject() + ":" + sub1.getName() + ":" + sub1.getExclude(); - - // Case: explicit setting of exclude to true - String actual2 = Store.parseSubscriptionFrom(allSubscriptions.get(3)); - Subscription sub2 = allSubscriptions.get(3); - String expected2 = sub2.getProject() + ":" + sub2.getName() + ":" + sub2.getExclude(); - - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - } - - @Test - public void shouldSubscribeToFeatureSet() { - allSubscriptions = allSubscriptions.subList(2, 4); - // Case: excluded flag = true - boolean actual1 = Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_1"); - boolean expected1 = false; - - // Case: excluded flag = false - boolean actual2 = Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_2"); - boolean expected2 = true; - - // Case: featureset does not exist - boolean actual3 = - Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_nonexistent"); - boolean expected3 = false; - - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - assertThat(actual3, equalTo(expected3)); - } -} diff --git a/core/.gitignore b/core/.gitignore deleted file mode 100644 index fc240cb6e9a..00000000000 --- a/core/.gitignore +++ /dev/null @@ -1,40 +0,0 @@ -### Scratch files ### -scratch_redis* - -### Local Environment ### -*local*.env - -### Gradle ### -.gradle -**/build/ -!gradle/wrapper/gradle-wrapper.jar -feast-serving.jar - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### IntelliJ IDEA ### -.idea -*.iws -*.iml -*.ipr -/out/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -## Static site ## -static - -## Feast Temporary Files ## -/temp/ \ No newline at end of file diff --git a/core/README.md b/core/README.md deleted file mode 100644 index 8f5b6f03bf5..00000000000 --- a/core/README.md +++ /dev/null @@ -1,32 +0,0 @@ -### Getting Started Guide for Feast Core Developers - -Pre-requisites: -- [Maven](https://maven.apache.org/install.html) build tool version 3.6.x -- A running Postgres instance. For easier to get started, please configure the database like so - ``` - database: postgres - user: postgres - password: password - ``` -- A running Redis instance - ``` - host: localhost - port: 6379 - ``` -- Access to Google Cloud BigQuery (optional) -- Access to Kafka brokers (to test starting ingestion jobs from Feast Core) - -Run the following maven command to start Feast Core GRPC service running on port 6565 locally -```bash -# Using configuration from src/main/resources/application.yml -mvn spring-boot:run -# Using configuration from custom location e.g. /tmp/config.application.yml -mvn spring-boot:run -Dspring.config.location=/tmp/config.application.yml -``` - -If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) installed, you can check that Feast Core is running -``` -grpc_cli ls localhost:6565 -grpc_cli call localhost:6565 GetFeastCoreVersion "" -grpc_cli call localhost:6565 ListStores "" -``` \ No newline at end of file diff --git a/core/lombok.config b/core/lombok.config deleted file mode 100644 index 8f7e8aa1ac9..00000000000 --- a/core/lombok.config +++ /dev/null @@ -1 +0,0 @@ -lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml deleted file mode 100644 index 7a34b794db2..00000000000 --- a/core/pom.xml +++ /dev/null @@ -1,326 +0,0 @@ - - - - 4.0.0 - - dev.feast - feast-parent - ${revision} - - - Feast Core - Feature registry - feast-core - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - - - - - org.jacoco - jacoco-maven-plugin - - - - org.springframework.boot - spring-boot-maven-plugin - - false - - - - build-info - - build-info - - - - - - - org.flywaydb - flyway-maven-plugin - ${flyway.version} - - - - - - - dev.feast - feast-common - ${project.version} - - - dev.feast - feast-common-test - ${project.version} - test - - - - - org.springframework.boot - spring-boot-devtools - true - - - - javax.inject - javax.inject - 1 - - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-log4j2 - - - org.apache.logging.log4j - log4j-web - - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security.oauth - spring-security-oauth2 - ${spring.security.oauth2.version} - - - org.springframework.security - spring-security-oauth2-client - ${spring.security.version} - - - org.springframework.security - spring-security-web - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-resource-server - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - - - net.devh - grpc-server-spring-boot-starter - ${grpc.spring.boot.starter.version} - - - com.nimbusds - nimbus-jose-jwt - 8.2.1 - - - org.springframework.security - spring-security-oauth2-core - ${spring.security.version} - - - - org.springframework.boot - spring-boot-starter-data-jpa - - - - org.springframework.boot - spring-boot-starter-actuator - - - - - org.springframework.boot - spring-boot-configuration-processor - - - - io.grpc - grpc-services - - - - io.grpc - grpc-stub - - - - com.google.protobuf - protobuf-java-util - - - - com.google.guava - guava - - - - com.google.code.gson - gson - 2.8.5 - - - com.google.api-client - google-api-client - 1.30.9 - - - com.google.apis - google-api-services-dataflow - v1b3-rev20200305-1.30.9 - - - org.hibernate - hibernate-core - - - - - org.postgresql - postgresql - provided - true - - - - org.springframework.kafka - spring-kafka - - - - - org.projectlombok - lombok - ${lombok.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_servlet - - - com.google.api.client - google-api-client-googleapis-auth-oauth - 1.2.3-alpha - - - com.auth0 - jwks-rsa - 0.11.0 - - - - com.auth0 - java-jwt - 3.10.0 - - - - org.apache.commons - commons-lang3 - - - - joda-time - joda-time - - - - - com.jayway.jsonpath - json-path-assert - 2.2.0 - test - - - - javax.xml.bind - jaxb-api - - - org.flywaydb - flyway-core - ${flyway.version} - - - org.hibernate.validator - hibernate-validator-annotation-processor - 6.1.2.Final - - - - sh.ory.keto - keto-client - 0.4.4-alpha.1 - test - - - com.github.tomakehurst - wiremock - 2.27.0 - test - - - org.apache.avro - avro - 1.8.2 - test - - - com.squareup.okhttp - okhttp - 2.7.4 - test - - - io.grpc - grpc-testing - - - diff --git a/core/src/main/java/feast/core/CoreApplication.java b/core/src/main/java/feast/core/CoreApplication.java deleted file mode 100644 index 957fdf50157..00000000000 --- a/core/src/main/java/feast/core/CoreApplication.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core; - -import feast.core.config.FeastProperties; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.scheduling.annotation.EnableScheduling; - -@EnableScheduling -@SpringBootApplication -@EnableJpaRepositories(basePackages = "feast.core.dao") -@EnableConfigurationProperties(FeastProperties.class) -@Slf4j -public class CoreApplication { - public static void main(String[] args) { - SpringApplication.run(CoreApplication.class, args); - } -} diff --git a/core/src/main/java/feast/core/config/CoreSecurityConfig.java b/core/src/main/java/feast/core/config/CoreSecurityConfig.java deleted file mode 100644 index 52911c3b223..00000000000 --- a/core/src/main/java/feast/core/config/CoreSecurityConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.proto.core.CoreServiceGrpc; -import io.grpc.health.v1.HealthGrpc; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.security.check.AccessPredicate; -import net.devh.boot.grpc.server.security.check.GrpcSecurityMetadataSource; -import net.devh.boot.grpc.server.security.check.ManualGrpcSecurityMetadataSource; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Configuration -@Slf4j -@ComponentScan( - basePackages = { - "feast.common.auth.config", - "feast.common.auth.service", - "feast.common.logging.interceptors" - }) -public class CoreSecurityConfig { - - /** - * Creates a SecurityMetadataSource when authentication is enabled. This allows for the - * configuration of endpoint level security rules. - * - * @return GrpcSecurityMetadataSource - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcSecurityMetadataSource grpcSecurityMetadataSource() { - final ManualGrpcSecurityMetadataSource source = new ManualGrpcSecurityMetadataSource(); - - // Authentication is enabled for all gRPC endpoints - source.setDefault(AccessPredicate.authenticated()); - - // The following endpoints allow unauthenticated access - source.set(CoreServiceGrpc.getGetFeastCoreVersionMethod(), AccessPredicate.permitAll()); - source.set(CoreServiceGrpc.getUpdateStoreMethod(), AccessPredicate.permitAll()); - source.set(HealthGrpc.getCheckMethod(), AccessPredicate.permitAll()); - return source; - } -} diff --git a/core/src/main/java/feast/core/config/FeastProperties.java b/core/src/main/java/feast/core/config/FeastProperties.java deleted file mode 100644 index fd926ea5da4..00000000000 --- a/core/src/main/java/feast/core/config/FeastProperties.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.logging.config.LoggingProperties; -import feast.common.validators.OneOfStrings; -import feast.core.config.FeastProperties.StreamProperties.FeatureStreamOptions; -import java.util.Set; -import javax.annotation.PostConstruct; -import javax.validation.ConstraintViolation; -import javax.validation.ConstraintViolationException; -import javax.validation.Validation; -import javax.validation.Validator; -import javax.validation.ValidatorFactory; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.info.BuildProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@Configuration -@ComponentScan("feast.common.logging") -@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true) -public class FeastProperties { - - /** - * Instantiates a new Feast properties. - * - * @param buildProperties Feast build properties - */ - @Autowired - public FeastProperties(BuildProperties buildProperties) { - setVersion(buildProperties.getVersion()); - } - - /** Instantiates a new Feast properties. */ - public FeastProperties() {} - - /* Feast Core Build Version */ - @NotBlank private String version = "unknown"; - - @NotNull - /* Feast Kafka stream properties */ - private StreamProperties stream; - - @NotNull private SecurityProperties security; - - @Bean - SecurityProperties securityProperties() { - return getSecurity(); - } - - /* Feast Audit Logging properties */ - @NotNull private LoggingProperties logging; - - @Bean - LoggingProperties loggingProperties() { - return getLogging(); - } - - /** Properties used to configure Feast's managed Kafka feature stream. */ - @Getter - @Setter - public static class StreamProperties { - - /* Feature stream type. Only "kafka" is supported. */ - @OneOfStrings({"kafka"}) - @NotBlank - private String type; - - /* Feature stream options */ - @NotNull private FeatureStreamOptions options; - - /** Feature stream options */ - @Getter - @Setter - public static class FeatureStreamOptions { - - /* Kafka topic to use for feature sets without source topics. */ - @NotBlank private String topic = "feast-features"; - - /** - * Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined - * source. - */ - @NotBlank private String bootstrapServers = "localhost:9092"; - - /* Defines the number of copies of managed feature stream Kafka. */ - @Positive private short replicationFactor = 1; - - /* Number of Kafka partitions to to use for managed feature stream. */ - @Positive private int partitions = 1; - } - } - - /** - * Validates all FeastProperties. This method runs after properties have been initialized and - * individually and conditionally validates each class. - */ - @PostConstruct - public void validate() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - - // Validate root fields in FeastProperties - Set> violations = validator.validate(this); - if (!violations.isEmpty()) { - throw new ConstraintViolationException(violations); - } - - // Validate Stream properties - Set> streamPropertyViolations = - validator.validate(getStream()); - if (!streamPropertyViolations.isEmpty()) { - throw new ConstraintViolationException(streamPropertyViolations); - } - - // Validate Stream Options - Set> featureStreamOptionsViolations = - validator.validate(getStream().getOptions()); - if (!featureStreamOptionsViolations.isEmpty()) { - throw new ConstraintViolationException(featureStreamOptionsViolations); - } - - // Validate AuthenticationProperties - Set> authenticationPropsViolations = - validator.validate(getSecurity().getAuthentication()); - if (!authenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authenticationPropsViolations); - } - - // Validate AuthorizationProperties - Set> authorizationPropsViolations = - validator.validate(getSecurity().getAuthorization()); - if (!authorizationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authorizationPropsViolations); - } - } -} diff --git a/core/src/main/java/feast/core/config/JPAConfig.java b/core/src/main/java/feast/core/config/JPAConfig.java deleted file mode 100644 index dada6c9d3a6..00000000000 --- a/core/src/main/java/feast/core/config/JPAConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import javax.persistence.EntityManagerFactory; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.transaction.PlatformTransactionManager; - -/** Configuration of JPA related services and beans for the core application. */ -@Configuration -@Slf4j -public class JPAConfig { - @Bean - public PlatformTransactionManager transactionManager(EntityManagerFactory emf) { - JpaTransactionManager transactionManager = new JpaTransactionManager(); - transactionManager.setEntityManagerFactory(emf); - - return transactionManager; - } -} diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java deleted file mode 100644 index 5fc6b8280e5..00000000000 --- a/core/src/main/java/feast/core/config/MonitoringConfig.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.StoreRepository; -import feast.core.metrics.collector.FeastResourceCollector; -import feast.core.metrics.collector.JVMResourceCollector; -import io.prometheus.client.exporter.MetricsServlet; -import javax.servlet.http.HttpServlet; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.web.servlet.ServletRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class MonitoringConfig { - - private static final String PROMETHEUS_METRICS_PATH = "/metrics"; - - /** - * Add Prometheus exposition to an existing HTTP server using servlets. - * - *

https://github.com/prometheus/client_java/tree/b61dd232a504e20dad404a2bf3e2c0b8661c212a#http - * - * @return HTTP servlet for returning metrics data - */ - @Bean - public ServletRegistrationBean metricsServlet() { - return new ServletRegistrationBean<>(new MetricsServlet(), PROMETHEUS_METRICS_PATH); - } - - /** - * Register custom Prometheus collector that exports metrics about Feast Resources. - * - *

For example: total number of registered feature tables and stores. - * - * @param featureTableRepository {@link FeatureTableRepository} - * @param storeRepository {@link StoreRepository} - * @return {@link FeastResourceCollector} - */ - @Bean - @Autowired - public FeastResourceCollector feastResourceCollector( - FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { - FeastResourceCollector collector = - new FeastResourceCollector(featureTableRepository, storeRepository); - collector.register(); - return collector; - } - - /** - * Register custom Prometheus collector that exports metrics about JVM resource usage. - * - * @return {@link JVMResourceCollector} - */ - @Bean - public JVMResourceCollector jvmResourceCollector() { - JVMResourceCollector jvmResourceCollector = new JVMResourceCollector(); - jvmResourceCollector.register(); - return jvmResourceCollector; - } -} diff --git a/core/src/main/java/feast/core/config/WebMvcConfig.java b/core/src/main/java/feast/core/config/WebMvcConfig.java deleted file mode 100644 index 5d6a6b8eceb..00000000000 --- a/core/src/main/java/feast/core/config/WebMvcConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import java.util.List; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.http.converter.protobuf.ProtobufJsonFormatHttpMessageConverter; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -/** Configuration for the spring web MVC layer */ -@Configuration -public class WebMvcConfig implements WebMvcConfigurer { - /** - * Get a json-protobuf converter. - * - * @return ProtobufJsonFormatHttpMessageConverter - */ - @Bean - ProtobufJsonFormatHttpMessageConverter getProtobufHttpMessageConverter() { - return new ProtobufJsonFormatHttpMessageConverter(); - } - - /** Register json-protobuf converter. */ - @Override - public void configureMessageConverters(List> converters) { - converters.add(getProtobufHttpMessageConverter()); - } -} diff --git a/core/src/main/java/feast/core/config/WebSecurityConfig.java b/core/src/main/java/feast/core/config/WebSecurityConfig.java deleted file mode 100644 index 0f481115a2f..00000000000 --- a/core/src/main/java/feast/core/config/WebSecurityConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import java.util.ArrayList; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; - -/** - * WebSecurityConfig disables auto configuration of Spring HTTP Security and allows security methods - * to be overridden - */ -@Configuration -@EnableWebSecurity -public class WebSecurityConfig extends WebSecurityConfigurerAdapter { - - private final FeastProperties feastProperties; - - @Autowired - public WebSecurityConfig(FeastProperties feastProperties) { - this.feastProperties = feastProperties; - } - - /** - * Allows for custom web security rules to be applied. - * - * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception - */ - @Override - protected void configure(HttpSecurity http) throws Exception { - List matchersToBypass = new ArrayList<>(List.of("/actuator/**", "/metrics/**")); - - if (feastProperties.securityProperties().isDisableRestControllerAuth()) { - matchersToBypass.add("/api/v1/**"); - matchersToBypass.add("/api/v2/**"); - } - - // Bypasses security/authentication for the following paths - http.authorizeRequests() - .antMatchers(matchersToBypass.toArray(new String[0])) - .permitAll() - .anyRequest() - .authenticated() - .and() - .csrf() - .disable(); - } -} diff --git a/core/src/main/java/feast/core/controller/CoreServiceRestController.java b/core/src/main/java/feast/core/controller/CoreServiceRestController.java deleted file mode 100644 index f782c172cd6..00000000000 --- a/core/src/main/java/feast/core/controller/CoreServiceRestController.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller; - -import feast.core.config.FeastProperties; -import feast.core.model.Project; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse; -import feast.proto.core.CoreServiceProto.ListEntitiesRequest; -import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListFeaturesRequest; -import feast.proto.core.CoreServiceProto.ListFeaturesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -/** - * EXPERIMENTAL: Controller for HTTP endpoints to Feast Core. These endpoints are subject to change. - */ -@RestController -@CrossOrigin -@Slf4j -@RequestMapping(value = "/api", produces = "application/json") -public class CoreServiceRestController { - - private final FeastProperties feastProperties; - private SpecService specService; - private ProjectService projectService; - - @Autowired - public CoreServiceRestController( - FeastProperties feastProperties, SpecService specService, ProjectService projectService) { - this.feastProperties = feastProperties; - this.specService = specService; - this.projectService = projectService; - } - - /** - * GET /version : Fetches the version of Feast Core. - * - * @return (200 OK) Returns {@link GetFeastCoreVersionResponse} in JSON. - */ - @RequestMapping(value = "/v2/version", method = RequestMethod.GET) - public GetFeastCoreVersionResponse getVersion() { - GetFeastCoreVersionResponse response = - GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build(); - return response; - } - - /** - * GET /features : List Features based on project and entities. - * - * @param entities Request Parameter: List of all entities every returned feature should belong - * to. At least one entity is required. For example, if entity1 and entity2 - * are given, then all features returned (if any) will belong to BOTH - * entities. - * @param project (Optional) Request Parameter: A single project where the feature table of all - * features returned is under. If not provided, the default project will be used, usually - * default. - * @return (200 OK) Return {@link ListFeaturesResponse} in JSON. - */ - @RequestMapping(value = "/v2/features", method = RequestMethod.GET) - public ListFeaturesResponse listFeatures( - @RequestParam String[] entities, @RequestParam(required = false) Optional project) { - ListFeaturesRequest.Filter.Builder filterBuilder = - ListFeaturesRequest.Filter.newBuilder().addAllEntities(Arrays.asList(entities)); - project.ifPresent(filterBuilder::setProject); - return specService.listFeatures(filterBuilder.build()); - } - - /** - * GET /projects : Get the list of existing feast projects. - * - * @return (200 OK) Returns {@link ListProjectsResponse} in JSON. - */ - @RequestMapping(value = "/v2/projects", method = RequestMethod.GET) - public ListProjectsResponse listProjects() { - List projects = projectService.listProjects(); - return ListProjectsResponse.newBuilder() - .addAllProjects(projects.stream().map(Project::getName).collect(Collectors.toList())) - .build(); - } - - /** - * GET /entities : Retrieve a list of Entities according to filtering parameters of Feast project - * name. If none matches, an empty JSON response is returned. - * - * @param project Request Parameter: Name of feast project to search in. - * @return (200 OK) Return {@link ListEntitiesResponse} in JSON. - */ - @RequestMapping(value = "/v2/entities", method = RequestMethod.GET) - public ListEntitiesResponse listEntities( - @RequestParam(defaultValue = Project.DEFAULT_NAME) String project) { - ListEntitiesRequest.Filter.Builder filterBuilder = - ListEntitiesRequest.Filter.newBuilder().setProject(project); - return specService.listEntities(filterBuilder.build()); - } - - /** - * GET /feature-tables : Retrieve a list of Feature Tables according to filtering parameters of - * Feast project name. If none matches, an empty JSON response is returned. - * - * @param project Request Parameter: Name of feast project to search in. - * @return (200 OK) Return {@link ListFeatureTablesResponse} in JSON. - */ - @RequestMapping(value = "/v2/feature-tables", method = RequestMethod.GET) - public ListFeatureTablesResponse listFeatureTables( - @RequestParam(defaultValue = Project.DEFAULT_NAME) String project) { - ListFeatureTablesRequest.Filter.Builder filterBuilder = - ListFeatureTablesRequest.Filter.newBuilder().setProject(project); - return specService.listFeatureTables(filterBuilder.build()); - } -} diff --git a/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java b/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java deleted file mode 100644 index ef27e4aee8a..00000000000 --- a/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller.exception.handler; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.exception.RetrievalException; -import java.util.Map; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.MissingServletRequestParameterException; -import org.springframework.web.bind.UnsatisfiedServletRequestParameterException; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; - -/** A exception handler for some common exceptions while accessing Feast Core via HTTP. */ -@ControllerAdvice -public class RestResponseEntityExceptionHandler extends ResponseEntityExceptionHandler { - - /** - * Handles the case when a request object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureTableRequest}) or a response object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureTableResponse} is malformed. - * - * @param ex the {@link InvalidProtocolBufferException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (500 Internal Server Error) - */ - @ExceptionHandler({InvalidProtocolBufferException.class}) - protected ResponseEntity handleInvalidProtocolBuffer( - InvalidProtocolBufferException ex, WebRequest request) { - Map bodyOfResponse = - Map.of("error", "An unexpected error occurred in Feast Core."); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.INTERNAL_SERVER_ERROR, request); - } - - /** - * Handles the case that retrieval of information from the services triggered and exception. - * Instead of returning 500 with no error message, returns 500 with a body describing the error - * message. - * - * @param ex the {@link RetrievalException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (500 Internal Server Error) - */ - @ExceptionHandler({RetrievalException.class}) - protected ResponseEntity handleRetrieval(RetrievalException ex, WebRequest request) { - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.INTERNAL_SERVER_ERROR, request); - } - - /** - * Handles various exceptions that are due to malformed or invalid requests, such as - * - *
    - *
  • {@link UnsatisfiedServletRequestParameterException} where a parameter is requested in - * {@link org.springframework.web.bind.annotation.RequestMapping} but not supplied. - *
  • {@link IllegalArgumentException} where unsupported parameters are provided. - *
- * - * @param ex the {@link UnsatisfiedServletRequestParameterException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (400 Bad Request) - */ - @ExceptionHandler({ - UnsatisfiedServletRequestParameterException.class, - IllegalArgumentException.class - }) - protected ResponseEntity handleBadRequest(Exception ex, WebRequest request) { - ex.printStackTrace(); - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.BAD_REQUEST, request); - } - - /** - * Handles {@link MissingServletRequestParameterException} which occurs when a controller method - * expects a certain parameter but is not supplied by the request. The original implementation - * returns an empty body with (400 Bad Request), we add in the error and stacktrace. - * - * @param ex the {@link MissingServletRequestParameterException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @param headers the {@link HttpHeaders} from the request. - * @param status the {@link HttpStatus} generated for the response, (400 Bad Request) - * @return (400 Bad Request) - */ - @Override - protected ResponseEntity handleMissingServletRequestParameter( - MissingServletRequestParameterException ex, - HttpHeaders headers, - HttpStatus status, - WebRequest request) { - ex.printStackTrace(); - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return this.handleExceptionInternal(ex, bodyOfResponse, headers, status, request); - } -} diff --git a/core/src/main/java/feast/core/dao/EntityRepository.java b/core/src/main/java/feast/core/dao/EntityRepository.java deleted file mode 100644 index d7d7dcb5b9f..00000000000 --- a/core/src/main/java/feast/core/dao/EntityRepository.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.EntityV2; -import java.util.List; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying EntityV2 objects keyed by id. */ -public interface EntityRepository extends JpaRepository { - - long count(); - - // Find all EntityV2s by project - List findAllByProject_Name(String project); - - // Find single EntityV2 by project and name - EntityV2 findEntityByNameAndProject_Name(String name, String project); -} diff --git a/core/src/main/java/feast/core/dao/FeatureTableRepository.java b/core/src/main/java/feast/core/dao/FeatureTableRepository.java deleted file mode 100644 index 2d125345a92..00000000000 --- a/core/src/main/java/feast/core/dao/FeatureTableRepository.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.FeatureTable; -import java.util.List; -import java.util.Optional; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository for querying FeatureTables stored. */ -public interface FeatureTableRepository extends JpaRepository { - // Find single FeatureTable by project and name - Optional findFeatureTableByNameAndProject_Name(String name, String projectName); - - // Find FeatureTables by project - List findAllByProject_Name(String projectName); -} diff --git a/core/src/main/java/feast/core/dao/ProjectRepository.java b/core/src/main/java/feast/core/dao/ProjectRepository.java deleted file mode 100644 index 5adb7d44c2e..00000000000 --- a/core/src/main/java/feast/core/dao/ProjectRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Project; -import java.util.List; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Project objects keyed by id. */ -public interface ProjectRepository extends JpaRepository { - - List findAllByArchivedIsFalse(); -} diff --git a/core/src/main/java/feast/core/dao/StoreRepository.java b/core/src/main/java/feast/core/dao/StoreRepository.java deleted file mode 100644 index 2cc00704711..00000000000 --- a/core/src/main/java/feast/core/dao/StoreRepository.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Store; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Store objects keyed by id. */ -public interface StoreRepository extends JpaRepository { - long count(); -} diff --git a/core/src/main/java/feast/core/exception/RegistrationException.java b/core/src/main/java/feast/core/exception/RegistrationException.java deleted file mode 100644 index ccb4d55caef..00000000000 --- a/core/src/main/java/feast/core/exception/RegistrationException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.exception; - -/** Exception thrown when a spec is fails to be registered to the Feast metadata registry. */ -public class RegistrationException extends RuntimeException { - public RegistrationException() { - super(); - } - - public RegistrationException(String message) { - super(message); - } - - public RegistrationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/core/src/main/java/feast/core/exception/RetrievalException.java b/core/src/main/java/feast/core/exception/RetrievalException.java deleted file mode 100644 index bd543048a98..00000000000 --- a/core/src/main/java/feast/core/exception/RetrievalException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.exception; - -/** Exception thrown when retrieval of a spec from the registry fails. */ -public class RetrievalException extends RuntimeException { - public RetrievalException() { - super(); - } - - public RetrievalException(String message) { - super(message); - } - - public RetrievalException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java deleted file mode 100644 index efdf0fc778e..00000000000 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc; - -import feast.common.auth.service.AuthorizationService; -import feast.common.logging.interceptors.GrpcMessageInterceptor; -import feast.core.config.FeastProperties; -import feast.core.exception.RetrievalException; -import feast.core.grpc.interceptors.MonitoringInterceptor; -import feast.core.model.Project; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase; -import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.EntityProto.EntitySpecV2; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.context.SecurityContextHolder; - -/** Implementation of the feast core GRPC service. */ -@Slf4j -@GrpcService(interceptors = {GrpcMessageInterceptor.class, MonitoringInterceptor.class}) -public class CoreServiceImpl extends CoreServiceImplBase { - - private final FeastProperties feastProperties; - private SpecService specService; - private ProjectService projectService; - private final AuthorizationService authorizationService; - - @Autowired - public CoreServiceImpl( - SpecService specService, - ProjectService projectService, - FeastProperties feastProperties, - AuthorizationService authorizationService) { - this.specService = specService; - this.projectService = projectService; - this.feastProperties = feastProperties; - this.authorizationService = authorizationService; - } - - @Override - public void getFeastCoreVersion( - GetFeastCoreVersionRequest request, - StreamObserver responseObserver) { - try { - GetFeastCoreVersionResponse response = - GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException | StatusRuntimeException e) { - log.error("Could not determine Feast Core version: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void getEntity( - GetEntityRequest request, StreamObserver responseObserver) { - try { - GetEntityResponse response = specService.getEntity(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException e) { - log.error("Unable to fetch entity requested in GetEntity method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to GetEntity method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in GetEntity method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /** Retrieve a list of features */ - @Override - public void listFeatures( - ListFeaturesRequest request, StreamObserver responseObserver) { - try { - ListFeaturesResponse response = specService.listFeatures(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to ListFeatures method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (RetrievalException e) { - log.error("Unable to fetch entities requested in ListFeatures method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ListFeatures method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /** Retrieve a list of entities */ - @Override - public void listEntities( - ListEntitiesRequest request, StreamObserver responseObserver) { - try { - ListEntitiesResponse response = specService.listEntities(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to ListEntities method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (RetrievalException e) { - log.error("Unable to fetch entities requested in ListEntities method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ListEntities method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listStores( - ListStoresRequest request, StreamObserver responseObserver) { - try { - ListStoresResponse response = specService.listStores(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException e) { - log.error("Exception has occurred in ListStores method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /* Registers an entity to Feast Core */ - @Override - public void applyEntity( - ApplyEntityRequest request, StreamObserver responseObserver) { - - String projectId = null; - - try { - EntitySpecV2 spec = request.getSpec(); - projectId = request.getProject(); - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectId); - ApplyEntityResponse response = specService.applyEntity(spec, projectId); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (org.hibernate.exception.ConstraintViolationException e) { - log.error( - "Unable to persist this entity due to a constraint violation. Please ensure that" - + " field names are unique within the project namespace: ", - e); - responseObserver.onError( - Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing project: %s", projectId)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ApplyEntity method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void updateStore( - UpdateStoreRequest request, StreamObserver responseObserver) { - try { - UpdateStoreResponse response = specService.updateStore(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in UpdateStore method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void createProject( - CreateProjectRequest request, StreamObserver responseObserver) { - try { - projectService.createProject(request.getName()); - responseObserver.onNext(CreateProjectResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in the createProject method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void archiveProject( - ArchiveProjectRequest request, StreamObserver responseObserver) { - String projectId = null; - try { - projectId = request.getName(); - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectId); - projectService.archiveProject(projectId); - responseObserver.onNext(ArchiveProjectResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Recieved an invalid request on calling archiveProject method:", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (UnsupportedOperationException e) { - log.error("Attempted to archive an unsupported project:", e); - responseObserver.onError( - Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing project: %s", projectId)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in the createProject method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listProjects( - ListProjectsRequest request, StreamObserver responseObserver) { - try { - List projects = projectService.listProjects(); - responseObserver.onNext( - ListProjectsResponse.newBuilder() - .addAllProjects(projects.stream().map(Project::getName).collect(Collectors.toList())) - .build()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in the listProjects method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void applyFeatureTable( - ApplyFeatureTableRequest request, - StreamObserver responseObserver) { - String projectName = SpecService.resolveProjectName(request.getProject()); - String tableName = request.getTableSpec().getName(); - - try { - // Check if user has authorization to apply feature table - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectName); - - ApplyFeatureTableResponse response = specService.applyFeatureTable(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (AccessDeniedException e) { - log.info( - String.format( - "ApplyFeatureTable: Not authorized to access project to apply: %s", projectName)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (org.hibernate.exception.ConstraintViolationException e) { - log.error( - String.format( - "ApplyFeatureTable: Unable to apply Feature Table due to a conflict: " - + "Ensure that name is unique within Project: (name: %s, project: %s)", - projectName, tableName)); - responseObserver.onError( - Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (IllegalArgumentException e) { - log.error( - String.format( - "ApplyFeatureTable: Invalid apply Feature Table Request: (name: %s, project: %s)", - projectName, tableName)); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (UnsupportedOperationException e) { - log.error( - String.format( - "ApplyFeatureTable: Unsupported apply Feature Table Request: (name: %s, project: %s)", - projectName, tableName)); - responseObserver.onError( - Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("ApplyFeatureTable Exception has occurred:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listFeatureTables( - ListFeatureTablesRequest request, - StreamObserver responseObserver) { - try { - ListFeatureTablesResponse response = specService.listFeatureTables(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error(String.format("ListFeatureTable: Invalid list Feature Table Request")); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("ListFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void getFeatureTable( - GetFeatureTableRequest request, StreamObserver responseObserver) { - try { - GetFeatureTableResponse response = specService.getFeatureTable(request); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - String.format( - "GetFeatureTable: No such Feature Table: (project: %s, name: %s)", - request.getProject(), request.getName())); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("GetFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void deleteFeatureTable( - DeleteFeatureTableRequest request, - StreamObserver responseObserver) { - String projectName = request.getProject(); - try { - // Check if user has authorization to delete feature table - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectName); - specService.deleteFeatureTable(request); - - responseObserver.onNext(DeleteFeatureTableResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - String.format( - "DeleteFeatureTable: No such Feature Table: (project: %s, name: %s)", - request.getProject(), request.getName())); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("DeleteFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/core/src/main/java/feast/core/grpc/HealthServiceImpl.java b/core/src/main/java/feast/core/grpc/HealthServiceImpl.java deleted file mode 100644 index ed6e0a1c728..00000000000 --- a/core/src/main/java/feast/core/grpc/HealthServiceImpl.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc; - -import feast.core.service.ProjectService; -import io.grpc.Status; -import io.grpc.health.v1.HealthGrpc.HealthImplBase; -import io.grpc.health.v1.HealthProto.HealthCheckRequest; -import io.grpc.health.v1.HealthProto.HealthCheckResponse; -import io.grpc.health.v1.HealthProto.ServingStatus; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; - -@Slf4j -@GrpcService -public class HealthServiceImpl extends HealthImplBase { - private final ProjectService projectService; - - @Autowired - public HealthServiceImpl(ProjectService projectService) { - this.projectService = projectService; - } - - @Override - public void check( - HealthCheckRequest request, StreamObserver responseObserver) { - try { - projectService.listProjects(); - responseObserver.onNext( - HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Health Check: unable to retrieve projects.\nError: %s", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java b/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java deleted file mode 100644 index 870675594a7..00000000000 --- a/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc.interceptors; - -import feast.core.metrics.GrpcMetrics; -import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.ServerCall; -import io.grpc.ServerCall.Listener; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; - -/** - * MonitoringInterceptor intercepts a GRPC call to provide a request latency historgram metrics in - * the Prometheus client. - */ -public class MonitoringInterceptor implements ServerInterceptor { - - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - - long startCallMillis = System.currentTimeMillis(); - String fullMethodName = call.getMethodDescriptor().getFullMethodName(); - String serviceName = MethodDescriptor.extractFullServiceName(fullMethodName); - String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); - - return next.startCall( - new SimpleForwardingServerCall(call) { - @Override - public void close(Status status, Metadata trailers) { - GrpcMetrics.requestLatency - .labels(serviceName, methodName, status.getCode().name()) - .observe((System.currentTimeMillis() - startCallMillis) / 1000f); - super.close(status, trailers); - } - }, - headers); - } -} diff --git a/core/src/main/java/feast/core/metrics/GrpcMetrics.java b/core/src/main/java/feast/core/metrics/GrpcMetrics.java deleted file mode 100644 index 7c20e632506..00000000000 --- a/core/src/main/java/feast/core/metrics/GrpcMetrics.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics; - -import io.prometheus.client.Histogram; - -public class GrpcMetrics { - - public static final Histogram requestLatency = - Histogram.build() - .name("feast_core_request_latency_seconds") - .labelNames("service", "method", "status_code") - .help("Request latency in seconds") - .register(); -} diff --git a/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java b/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java deleted file mode 100644 index 3064a25b8f7..00000000000 --- a/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics.collector; - -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.StoreRepository; -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; -import java.util.ArrayList; -import java.util.List; - -/** - * FeastResourceCollector exports metrics about Feast Resources. - * - *

For example: total number of registered feature tables and stores. - */ -public class FeastResourceCollector extends Collector { - - private final FeatureTableRepository featureTableRepository; - private final StoreRepository storeRepository; - - public FeastResourceCollector( - FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { - this.featureTableRepository = featureTableRepository; - this.storeRepository = storeRepository; - } - - @Override - public List collect() { - List samples = new ArrayList<>(); - samples.add( - new GaugeMetricFamily( - "feast_core_feature_set_total", - "Total number of registered feature tables", - featureTableRepository.count())); - samples.add( - new GaugeMetricFamily( - "feast_core_store_total", - "Total number of registered stores", - storeRepository.count())); - return samples; - } -} diff --git a/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java b/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java deleted file mode 100644 index 8602f6c249e..00000000000 --- a/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics.collector; - -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; -import io.prometheus.client.SummaryMetricFamily; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * JVMResourceCollector exports metrics about Java virtual machine memory and garbage collection. - */ -public class JVMResourceCollector extends Collector { - - private final List garbageCollectors; - private final Runtime runtime; - - public JVMResourceCollector() { - garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans(); - runtime = Runtime.getRuntime(); - } - - @Override - public List collect() { - List samples = new ArrayList<>(); - - samples.add( - new GaugeMetricFamily( - "feast_core_max_memory_bytes", - "Max amount of memory the Java virtual machine will attempt to use", - runtime.maxMemory())); - samples.add( - new GaugeMetricFamily( - "feast_core_total_memory_bytes", - "Total amount of memory in the Java virtual machine", - runtime.totalMemory())); - samples.add( - new GaugeMetricFamily( - "feast_core_free_memory_bytes", - "Total amount of free memory in the Java virtual machine", - runtime.freeMemory())); - - SummaryMetricFamily gcMetricFamily = - new SummaryMetricFamily( - "feast_core_gc_collection_seconds", - "Time spent in a given JVM garbage collector in seconds", - Collections.singletonList("gc")); - for (final GarbageCollectorMXBean gc : garbageCollectors) { - gcMetricFamily.addMetric( - Collections.singletonList(gc.getName()), - gc.getCollectionCount(), - gc.getCollectionTime() / MILLISECONDS_PER_SECOND); - } - samples.add(gcMetricFamily); - - return samples; - } -} diff --git a/core/src/main/java/feast/core/model/AbstractTimestampEntity.java b/core/src/main/java/feast/core/model/AbstractTimestampEntity.java deleted file mode 100644 index 188f19b5bc0..00000000000 --- a/core/src/main/java/feast/core/model/AbstractTimestampEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import java.time.Instant; -import java.util.Date; -import javax.persistence.*; -import lombok.Data; - -/** - * Base object class ensuring that all objects stored in the registry have an auto-generated - * creation and last updated time. - */ -@MappedSuperclass -@Data -public abstract class AbstractTimestampEntity { - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "created", nullable = false) - private Date created; - - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "last_updated", nullable = false) - private Date lastUpdated; - - @PrePersist - protected void onCreate() { - lastUpdated = created = new Date(); - } - - @PreUpdate - protected void onUpdate() { - lastUpdated = new Date(); - } - - // This constructor is used for testing. - public AbstractTimestampEntity() { - this.created = Date.from(Instant.ofEpochMilli(0L)); - this.lastUpdated = Date.from(Instant.ofEpochMilli(0L)); - } -} diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java deleted file mode 100644 index 67477dab45a..00000000000 --- a/core/src/main/java/feast/core/model/DataSource.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.MessageOrBuilder; -import com.google.protobuf.util.JsonFormat; -import feast.core.util.TypeConversion; -import feast.proto.core.DataFormatProto.FileFormat; -import feast.proto.core.DataFormatProto.StreamFormat; -import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; -import feast.proto.core.DataSourceProto.DataSource.FileOptions; -import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import java.util.HashMap; -import java.util.Map; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.Table; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -@Entity -@Getter -@Setter(AccessLevel.PRIVATE) -@Table(name = "data_sources") -public class DataSource { - @Column(name = "id") - @Id - @GeneratedValue - private long id; - - // Type of this Data Source - @Enumerated(EnumType.STRING) - @Column(name = "type", nullable = false) - private SourceType type; - - // DataSource Options - @Column(name = "config") - private String configJSON; - - // Field mapping between sourced fields (key) and feature fields (value). - // Stored as serialized JSON string. - @Column(name = "field_mapping", columnDefinition = "text") - private String fieldMapJSON; - - @Column(name = "timestamp_column") - private String eventTimestampColumn; - - @Column(name = "created_timestamp_column") - private String createdTimestampColumn; - - @Column(name = "date_partition_column") - private String datePartitionColumn; - - public DataSource() {}; - - public DataSource(SourceType type) { - this.type = type; - } - - /** - * Construct a DataSource from the given Protobuf representation spec - * - * @param spec Protobuf representation of DataSource to construct from. - * @throws IllegalArgumentException when provided with a invalid Protobuf spec - * @throws UnsupportedOperationException if source type is unsupported. - */ - public static DataSource fromProto(DataSourceProto.DataSource spec) { - DataSource source = new DataSource(spec.getType()); - // Copy source type specific options - Map dataSourceConfigMap = new HashMap<>(); - switch (spec.getType()) { - case BATCH_FILE: - dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); - dataSourceConfigMap.put("file_format", printJSON(spec.getFileOptions().getFileFormat())); - break; - case BATCH_BIGQUERY: - dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); - break; - case STREAM_KAFKA: - dataSourceConfigMap.put("bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); - dataSourceConfigMap.put( - "message_format", printJSON(spec.getKafkaOptions().getMessageFormat())); - dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); - break; - case STREAM_KINESIS: - dataSourceConfigMap.put( - "record_format", printJSON(spec.getKinesisOptions().getRecordFormat())); - dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); - dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); - - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); - } - - // Store DataSource mapping as serialised JSON - source.setConfigJSON(TypeConversion.convertMapToJsonString(dataSourceConfigMap)); - - // Store field mapping as serialised JSON - source.setFieldMapJSON(TypeConversion.convertMapToJsonString(spec.getFieldMappingMap())); - - // Set timestamp mapping columns - source.setEventTimestampColumn(spec.getEventTimestampColumn()); - source.setCreatedTimestampColumn(spec.getCreatedTimestampColumn()); - source.setDatePartitionColumn(spec.getDatePartitionColumn()); - - return source; - } - - /** Convert this DataSource to its Protobuf representation. */ - public DataSourceProto.DataSource toProto() { - DataSourceProto.DataSource.Builder spec = DataSourceProto.DataSource.newBuilder(); - spec.setType(getType()); - - // Extract source type specific options - Map dataSourceConfigMap = - TypeConversion.convertJsonStringToMap(getConfigJSON()); - switch (getType()) { - case BATCH_FILE: - FileOptions.Builder fileOptions = FileOptions.newBuilder(); - fileOptions.setFileUrl(dataSourceConfigMap.get("file_url")); - - FileFormat.Builder fileFormat = FileFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("file_format"), fileFormat); - fileOptions.setFileFormat(fileFormat.build()); - - spec.setFileOptions(fileOptions.build()); - break; - case BATCH_BIGQUERY: - BigQueryOptions.Builder bigQueryOptions = BigQueryOptions.newBuilder(); - bigQueryOptions.setTableRef(dataSourceConfigMap.get("table_ref")); - spec.setBigqueryOptions(bigQueryOptions.build()); - break; - case STREAM_KAFKA: - KafkaOptions.Builder kafkaOptions = KafkaOptions.newBuilder(); - kafkaOptions.setBootstrapServers(dataSourceConfigMap.get("bootstrap_servers")); - kafkaOptions.setTopic(dataSourceConfigMap.get("topic")); - - StreamFormat.Builder messageFormat = StreamFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("message_format"), messageFormat); - kafkaOptions.setMessageFormat(messageFormat.build()); - - spec.setKafkaOptions(kafkaOptions.build()); - break; - case STREAM_KINESIS: - KinesisOptions.Builder kinesisOptions = KinesisOptions.newBuilder(); - kinesisOptions.setRegion(dataSourceConfigMap.get("region")); - kinesisOptions.setStreamName(dataSourceConfigMap.get("stream_name")); - - StreamFormat.Builder recordFormat = StreamFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("record_format"), recordFormat); - kinesisOptions.setRecordFormat(recordFormat.build()); - - spec.setKinesisOptions(kinesisOptions.build()); - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", getType())); - } - - // Parse field mapping and options from JSON - spec.putAllFieldMapping(TypeConversion.convertJsonStringToMap(getFieldMapJSON())); - - spec.setEventTimestampColumn(getEventTimestampColumn()); - spec.setCreatedTimestampColumn(getCreatedTimestampColumn()); - spec.setDatePartitionColumn(getDatePartitionColumn()); - - return spec.build(); - } - - public Map getFieldsMap() { - return TypeConversion.convertJsonStringToMap(getFieldMapJSON()); - } - - @Override - public int hashCode() { - return toProto().hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DataSource other = (DataSource) o; - return this.toProto().equals(other.toProto()); - } - - /** Print the given Message into its JSON string representation */ - private static String printJSON(MessageOrBuilder message) { - try { - return JsonFormat.printer().print(message); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering Proto to JSON", e); - } - } - - /** Parse the given Message in JSON representation into the given Message Builder */ - private static void parseMessage(String json, Message.Builder message) { - try { - JsonFormat.parser().merge(json, message); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering JSON to Proto", e); - } - } -} diff --git a/core/src/main/java/feast/core/model/EntityV2.java b/core/src/main/java/feast/core/model/EntityV2.java deleted file mode 100644 index aeb6728454a..00000000000 --- a/core/src/main/java/feast/core/model/EntityV2.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.Timestamp; -import feast.core.util.TypeConversion; -import feast.proto.core.EntityProto; -import feast.proto.core.EntityProto.*; -import feast.proto.types.ValueProto.ValueType; -import java.util.Map; -import javax.persistence.*; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@javax.persistence.Entity -@Table( - name = "entities_v2", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) -public class EntityV2 extends AbstractTimestampEntity { - @Id @GeneratedValue private long id; - - // Name of the Entity - @Column(name = "name", nullable = false) - private String name; - - // Project that this Entity belongs to - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "project_name") - private Project project; - - // Description of entity - @Column(name = "description", columnDefinition = "text") - private String description; - - // Columns of entities - /** Data type of each entity column: String representation of {@link ValueType} * */ - private String type; - - // User defined metadata - @Column(name = "labels", columnDefinition = "text") - private String labels; - - public EntityV2() { - super(); - } - - /** - * EntityV2 object supports Entity registration in FeatureTable. - * - *

This data model supports Scalar Entity and would allow ease of discovery of entities and - * reasoning when used in association with FeatureTable. - */ - public EntityV2( - String name, String description, ValueType.Enum type, Map labels) { - this.name = name; - this.description = description; - this.type = type.toString(); - this.labels = TypeConversion.convertMapToJsonString(labels); - } - - public static EntityV2 fromProto(EntityProto.Entity entityProto) { - EntitySpecV2 spec = entityProto.getSpec(); - - return new EntityV2( - spec.getName(), spec.getDescription(), spec.getValueType(), spec.getLabelsMap()); - } - - public EntityProto.Entity toProto() { - EntityMeta.Builder meta = - EntityMeta.newBuilder() - .setCreatedTimestamp( - Timestamp.newBuilder().setSeconds(super.getCreated().getTime() / 1000L)) - .setLastUpdatedTimestamp( - Timestamp.newBuilder().setSeconds(super.getLastUpdated().getTime() / 1000L)); - - EntitySpecV2.Builder spec = - EntitySpecV2.newBuilder() - .setName(getName()) - .setDescription(getDescription()) - .setValueType(ValueType.Enum.valueOf(getType())) - .putAllLabels(TypeConversion.convertJsonStringToMap(labels)); - - // Build Entity - EntityProto.Entity entity = EntityProto.Entity.newBuilder().setMeta(meta).setSpec(spec).build(); - return entity; - } - - /** - * Updates the existing entity from a proto. - * - * @param entityProto EntityProto with updated spec - * @param projectName Project namespace of Entity which is to be created/updated - */ - public void updateFromProto(EntityProto.Entity entityProto, String projectName) { - EntitySpecV2 spec = entityProto.getSpec(); - - // Validate no change to type - if (!spec.getValueType().equals(ValueType.Enum.valueOf(getType()))) { - throw new IllegalArgumentException( - String.format( - "You are attempting to change the type of this entity in %s project from %s to %s. This isn't allowed. Please create a new entity.", - projectName, getType(), spec.getValueType().toString())); - } - - // 2. Update description, labels - this.setDescription(spec.getDescription()); - this.setLabels(TypeConversion.convertMapToJsonString(spec.getLabelsMap())); - } - - /** - * Determine whether an entity has all the specified labels. - * - * @param labelsFilter labels contain key-value mapping for labels attached to the Entity - * @return boolean True if Entity contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map LabelsMap = this.getLabelsMap(); - for (String key : labelsFilter.keySet()) { - if (!LabelsMap.containsKey(key) || !LabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public Map getLabelsMap() { - return TypeConversion.convertJsonStringToMap(this.getLabels()); - } -} diff --git a/core/src/main/java/feast/core/model/FeatureTable.java b/core/src/main/java/feast/core/model/FeatureTable.java deleted file mode 100644 index b442d57b583..00000000000 --- a/core/src/main/java/feast/core/model/FeatureTable.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static feast.common.models.FeatureV2.getFeatureStringRef; - -import com.google.common.hash.Hashing; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.core.dao.EntityRepository; -import feast.core.util.TypeConversion; -import feast.proto.core.DataSourceProto; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto; -import java.util.*; -import java.util.stream.Collectors; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.JoinTable; -import javax.persistence.ManyToMany; -import javax.persistence.ManyToOne; -import javax.persistence.OneToMany; -import javax.persistence.OneToOne; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Entity -@Setter(AccessLevel.PRIVATE) -@Table( - name = "feature_tables", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) -public class FeatureTable extends AbstractTimestampEntity { - - @Id @GeneratedValue private long id; - - // Name of Feature Table - @Column(name = "name", nullable = false) - private String name; - - // Name of the Project that this FeatureTable belongs to - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "project_name") - private Project project; - - // Features defined in this Feature Table - @OneToMany( - mappedBy = "featureTable", - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true) - private Set features; - - // Entites to associate the features defined in this FeatureTable with - @ManyToMany(fetch = FetchType.EAGER) - @JoinTable( - name = "feature_tables_entities_v2", - joinColumns = @JoinColumn(name = "feature_table_id"), - inverseJoinColumns = @JoinColumn(name = "entity_v2_id")) - private Set entities; - - // User defined metadata labels serialized as JSON string. - @Column(name = "labels", columnDefinition = "text") - private String labelsJSON; - - // Max Age of the Features defined in this Feature Table in seconds - @Column(name = "max_age", nullable = false) - private long maxAgeSecs; - - // Streaming DataSource used to obtain data for features from a stream - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "stream_source_id", nullable = true) - private DataSource streamSource; - - // Batched DataSource used to obtain data for features from a batch of data - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "batch_source_id", nullable = false) - private DataSource batchSource; - - // Autoincrementing version no. of this FeatureTable. - // Autoincrements every update made to the FeatureTable. - @Column(name = "revision", nullable = false) - private int revision; - - @Column(name = "is_deleted", nullable = false) - private boolean isDeleted; - - public FeatureTable() {}; - - /** - * Construct FeatureTable from Protobuf spec representation in the given project with entities - * registered in entity repository. - * - * @param projectName the name of the project that the constructed FeatureTable belongs. - * @param spec the Protobuf spec to construct the Feature from. - * @param entityRepo {@link EntityRepository} used to resolve entity names. - * @throws IllegalArgumentException if the Protobuf spec provided is invalid. - * @return constructed FeatureTable from the given Protobuf spec. - */ - public static FeatureTable fromProto( - String projectName, FeatureTableSpec spec, EntityRepository entityRepo) { - FeatureTable table = new FeatureTable(); - table.setName(spec.getName()); - table.setProject(new Project(projectName)); - - Set features = - spec.getFeaturesList().stream() - .map(featureSpec -> FeatureV2.fromProto(table, featureSpec)) - .collect(Collectors.toSet()); - table.setFeatures(features); - - Set entities = - FeatureTable.resolveEntities( - projectName, spec.getName(), entityRepo, spec.getEntitiesList()); - table.setEntities(entities); - - String labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - table.setLabelsJSON(labelsJSON); - - table.setMaxAgeSecs(spec.getMaxAge().getSeconds()); - table.setBatchSource(DataSource.fromProto(spec.getBatchSource())); - - // Configure stream source only if set - if (!spec.getStreamSource().equals(DataSourceProto.DataSource.getDefaultInstance())) { - table.setStreamSource(DataSource.fromProto(spec.getStreamSource())); - } - - return table; - } - - /** - * Update the FeatureTable from the given Protobuf representation. - * - * @param spec the Protobuf spec to update the FeatureTable from. - * @throws IllegalArgumentException if the update will make prohibited changes. - */ - public void updateFromProto( - String projectName, FeatureTableSpec spec, EntityRepository entityRepo) { - // Check for prohibited changes made in spec: - // - Name cannot be changed - if (!getName().equals(spec.getName())) { - throw new IllegalArgumentException( - String.format( - "Updating the name of a registered FeatureTable is not allowed: %s to %s", - getName(), spec.getName())); - } - // Update Entities if changed - Set entities = - FeatureTable.resolveEntities( - projectName, spec.getName(), entityRepo, spec.getEntitiesList()); - this.setEntities(entities); - - // Update FeatureTable based on spec - // Update existing features, create new feature, drop missing features - Map existingFeatures = - getFeatures().stream().collect(Collectors.toMap(FeatureV2::getName, feature -> feature)); - this.features.clear(); - this.features.addAll( - spec.getFeaturesList().stream() - .map( - featureSpec -> { - if (!existingFeatures.containsKey(featureSpec.getName())) { - // Create new Feature based on spec - return FeatureV2.fromProto(this, featureSpec); - } - // Update existing feature based on spec - FeatureV2 feature = existingFeatures.get(featureSpec.getName()); - feature.updateFromProto(featureSpec); - return feature; - }) - .collect(Collectors.toSet())); - - this.maxAgeSecs = spec.getMaxAge().getSeconds(); - this.labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - - this.batchSource = DataSource.fromProto(spec.getBatchSource()); - if (!spec.getStreamSource().equals(DataSourceProto.DataSource.getDefaultInstance())) { - this.streamSource = DataSource.fromProto(spec.getStreamSource()); - } else { - this.streamSource = null; - } - - // Set isDeleted to false - this.setDeleted(false); - - // Bump revision no. - this.revision++; - } - - /** Convert this Feature Table to its Protobuf representation */ - public FeatureTableProto.FeatureTable toProto() { - // Convert field types to Protobuf compatible types - Timestamp creationTime = TypeConversion.convertTimestamp(getCreated()); - Timestamp updatedTime = TypeConversion.convertTimestamp(getLastUpdated()); - String metadataHashBytes = this.protoHash(); - - List featureSpecs = - getFeatures().stream().map(FeatureV2::toProto).collect(Collectors.toList()); - List entityNames = - getEntities().stream().map(EntityV2::getName).collect(Collectors.toList()); - Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - - FeatureTableSpec.Builder spec = - FeatureTableSpec.newBuilder() - .setName(getName()) - .setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build()) - .setBatchSource(getBatchSource().toProto()) - .addAllEntities(entityNames) - .addAllFeatures(featureSpecs) - .putAllLabels(labels); - if (getStreamSource() != null) { - spec.setStreamSource(getStreamSource().toProto()); - } - - return FeatureTableProto.FeatureTable.newBuilder() - .setMeta( - FeatureTableProto.FeatureTableMeta.newBuilder() - .setRevision(getRevision()) - .setCreatedTimestamp(creationTime) - .setLastUpdatedTimestamp(updatedTime) - .setHash(metadataHashBytes) - .build()) - .setSpec(spec.build()) - .build(); - } - - /** Use given entity repository to resolve entity names to entity native objects */ - private static Set resolveEntities( - String projectName, String tableName, EntityRepository repo, Collection names) { - return names.stream() - .map( - entityName -> { - EntityV2 entity = repo.findEntityByNameAndProject_Name(entityName, projectName); - if (entity == null) { - throw new IllegalArgumentException( - String.format( - "Feature Table refers to no existent Entity: (table: %s, entity: %s, project: %s)", - tableName, entityName, projectName)); - } - return entity; - }) - .collect(Collectors.toSet()); - } - - /** - * Return a boolean to indicate if FeatureTable contains all specified entities. - * - * @param entitiesFilter contain entities that should be attached to the FeatureTable - * @return boolean True if FeatureTable contains all entities in the entitiesFilter - */ - public boolean hasAllEntities(List entitiesFilter) { - Set allEntitiesName = - this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toSet()); - return allEntitiesName.equals(new HashSet<>(entitiesFilter)); - } - - /** - * Returns a map of Feature references and Features if FeatureTable's Feature contains all labels - * in the labelsFilter - * - * @param labelsFilter contain labels that should be attached to FeatureTable's features - * @return Map of Feature references and Features - */ - public Map getFeaturesByLabels(Map labelsFilter) { - Map validFeaturesMap; - List validFeatures; - if (labelsFilter.size() > 0) { - validFeatures = filterFeaturesByAllLabels(this.getFeatures(), labelsFilter); - validFeaturesMap = getFeaturesRefToFeaturesMap(validFeatures); - return validFeaturesMap; - } - validFeaturesMap = getFeaturesRefToFeaturesMap(List.copyOf(this.getFeatures())); - return validFeaturesMap; - } - - /** - * Returns map for accessing features using their respective feature reference. - * - * @param features List of features to insert to map. - * @return Map of featureRef:feature. - */ - private Map getFeaturesRefToFeaturesMap(List features) { - Map validFeaturesMap = new HashMap<>(); - for (FeatureV2 feature : features) { - ServingAPIProto.FeatureReferenceV2 featureRef = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(this.getName()) - .setName(feature.getName()) - .build(); - validFeaturesMap.put(getFeatureStringRef(featureRef), feature); - } - return validFeaturesMap; - } - - /** - * Returns a list of Features if FeatureTable's Feature contains all labels in labelsFilter - * - * @param labelsFilter contain labels that should be attached to FeatureTable's features - * @return List of Features - */ - public static List filterFeaturesByAllLabels( - Set features, Map labelsFilter) { - List validFeatures = - features.stream() - .filter(feature -> feature.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - - return validFeatures; - } - - /** - * Determine whether a FeatureTable has all the specified labels. - * - * @param labelsFilter labels contain key-value mapping for labels attached to the FeatureTable - * @return boolean True if Entity contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map LabelsMap = this.getLabelsMap(); - for (String key : labelsFilter.keySet()) { - if (!LabelsMap.containsKey(key) || !LabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public Map getLabelsMap() { - return TypeConversion.convertJsonStringToMap(getLabelsJSON()); - } - - public void delete() { - this.setDeleted(true); - this.setRevision(0); - } - - public String protoHash() { - List sortedEntities = - this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toList()); - Collections.sort(sortedEntities); - - List sortedFeatures = new ArrayList(this.getFeatures()); - List sortedFeatureSpecs = - sortedFeatures.stream().map(featureV2 -> featureV2.toProto()).collect(Collectors.toList()); - sortedFeatures.sort(Comparator.comparing(FeatureV2::getName)); - - DataSourceProto.DataSource streamSource = DataSourceProto.DataSource.getDefaultInstance(); - if (getStreamSource() != null) { - streamSource = getStreamSource().toProto(); - } - - FeatureTableSpec featureTableSpec = - FeatureTableSpec.newBuilder() - .addAllEntities(sortedEntities) - .addAllFeatures(sortedFeatureSpecs) - .setBatchSource(getBatchSource().toProto()) - .setStreamSource(streamSource) - .setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build()) - .build(); - return Hashing.murmur3_32().hashBytes(featureTableSpec.toByteArray()).toString(); - } - - @Override - public int hashCode() { - return Objects.hash( - getName(), - getProject(), - getFeatures(), - getEntities(), - getMaxAgeSecs(), - getBatchSource(), - getStreamSource()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof FeatureTable)) { - return false; - } - - FeatureTable other = (FeatureTable) o; - - return getName().equals(other.getName()) - && getProject().equals(other.getProject()) - && getLabelsJSON().equals(other.getLabelsJSON()) - && getFeatures().equals(other.getFeatures()) - && getEntities().equals(other.getEntities()) - && getMaxAgeSecs() == other.getMaxAgeSecs() - && Optional.ofNullable(getBatchSource()).equals(Optional.ofNullable(other.getBatchSource())) - && Optional.ofNullable(getStreamSource()) - .equals(Optional.ofNullable(other.getStreamSource())); - } -} diff --git a/core/src/main/java/feast/core/model/FeatureV2.java b/core/src/main/java/feast/core/model/FeatureV2.java deleted file mode 100644 index f25e951efc7..00000000000 --- a/core/src/main/java/feast/core/model/FeatureV2.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import feast.core.util.TypeConversion; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.types.ValueProto.ValueType; -import java.util.Map; -import java.util.Objects; -import javax.persistence.*; -import javax.persistence.Entity; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -/** Defines a single Feature defined in a {@link FeatureTable} */ -@Getter -@Entity -@Setter(AccessLevel.PRIVATE) -@Table( - name = "features_v2", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_table_id"})) -public class FeatureV2 { - @Id @GeneratedValue private long id; - - // Name of the Feature - private String name; - - // Feature Table where this Feature is defined in. - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "feature_table_id", nullable = false) - private FeatureTable featureTable; - - // Value type of the feature. String representation of ValueType. - @Enumerated(EnumType.STRING) - @Column(name = "type") - private ValueType.Enum type; - - // User defined metadata labels for this feature encoded a JSON string. - @Column(name = "labels", columnDefinition = "text") - private String labelsJSON; - - public FeatureV2() {}; - - public FeatureV2(FeatureTable table, String name, ValueType.Enum type, String labelsJSON) { - this.featureTable = table; - this.name = name; - this.type = type; - this.labelsJSON = labelsJSON; - } - - /** - * Construct Feature from Protobuf spec representation. - * - * @param table the FeatureTable to associate the constructed feature with. - * @param spec the Protobuf spec to contruct the Feature from. - * @return constructed Feature from the given Protobuf spec. - */ - public static FeatureV2 fromProto(FeatureTable table, FeatureSpecV2 spec) { - String labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - return new FeatureV2(table, spec.getName(), spec.getValueType(), labelsJSON); - } - - /** Convert this Feature to its Protobuf representation. */ - public FeatureSpecV2 toProto() { - Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - return FeatureSpecV2.newBuilder() - .setName(getName()) - .setValueType(getType()) - .putAllLabels(labels) - .build(); - } - - /** - * Update the Feature from the given Protobuf representation. - * - * @param spec the Protobuf spec to update the Feature from. - * @throws IllegalArgumentException if the update will make prohibited changes. - */ - public void updateFromProto(FeatureSpecV2 spec) { - // Check for prohibited changes made in spec - if (!getName().equals(spec.getName())) { - throw new IllegalArgumentException( - String.format( - "Updating the name of a registered Feature is not allowed: %s to %s", - getName(), spec.getName())); - } - // Update feature type - this.setType(spec.getValueType()); - - // Update Feature based on spec - this.labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - } - - /** - * Return a boolean to indicate if Feature contains all specified labels. - * - * @param labelsFilter contain labels that should be attached to Feature - * @return boolean True if Feature contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map featureLabelsMap = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - for (String key : labelsFilter.keySet()) { - if (!featureLabelsMap.containsKey(key) - || !featureLabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - @Override - public int hashCode() { - return Objects.hash(getName(), getType(), getLabelsJSON()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FeatureV2 feature = (FeatureV2) o; - return getName().equals(feature.getName()) - && getType().equals(feature.getType()) - && getLabelsJSON().equals(feature.getLabelsJSON()); - } -} diff --git a/core/src/main/java/feast/core/model/Project.java b/core/src/main/java/feast/core/model/Project.java deleted file mode 100644 index 2d60d5e0e07..00000000000 --- a/core/src/main/java/feast/core/model/Project.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.OneToMany; -import javax.persistence.Table; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@Entity -@Table(name = "projects") -public class Project { - public static final String DEFAULT_NAME = "default"; - - // Name of the project - @Id - @Column(name = "name", nullable = false, unique = true) - private String name; - - // Flag to set whether the project has been archived - @Column(name = "archived", nullable = false) - private boolean archived; - - @OneToMany( - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true, - mappedBy = "project") - private Set entities; - - @OneToMany( - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true, - mappedBy = "project") - private Set featureTables; - - public Project() { - super(); - } - - public Project(String name) { - this.name = name; - this.entities = new HashSet<>(); - this.featureTables = new HashSet<>(); - } - - public void addEntity(EntityV2 entity) { - entity.setProject(this); - entities.add(entity); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Project field = (Project) o; - return name.equals(field.getName()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), name); - } -} diff --git a/core/src/main/java/feast/core/model/Store.java b/core/src/main/java/feast/core/model/Store.java deleted file mode 100644 index 7a8f9a6d61a..00000000000 --- a/core/src/main/java/feast/core/model/Store.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static feast.common.models.Store.convertStringToSubscription; -import static feast.common.models.Store.parseSubscriptionFrom; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.Builder; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.Table; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@AllArgsConstructor -@Entity -@Table(name = "stores") -public class Store { - - // Name of the store. Must be unique - @Id - @Column(name = "name", nullable = false, unique = true) - private String name; - - // Type of the store, should map to feast.core.Store.StoreType - @Column(name = "type", nullable = false) - private String type; - - // Connection string to the database - @Column(name = "config", nullable = false) - @Lob - private byte[] config; - - // FeatureSets this store is subscribed to, comma delimited. - @Column(name = "subscriptions") - private String subscriptions; - - public Store() { - super(); - } - - public static Store fromProto(StoreProto.Store storeProto) throws IllegalArgumentException { - List subs = new ArrayList<>(); - for (Subscription s : storeProto.getSubscriptionsList()) { - subs.add(parseSubscriptionFrom(s)); - } - byte[] config; - switch (storeProto.getType()) { - case REDIS: - config = storeProto.getRedisConfig().toByteArray(); - break; - case REDIS_CLUSTER: - config = storeProto.getRedisClusterConfig().toByteArray(); - break; - default: - throw new IllegalArgumentException("Invalid store provided"); - } - return new Store( - storeProto.getName(), storeProto.getType().toString(), config, String.join(",", subs)); - } - - public StoreProto.Store toProto() throws InvalidProtocolBufferException { - List subscriptionProtos = getSubscriptions(); - Builder storeProtoBuilder = - StoreProto.Store.newBuilder() - .setName(name) - .setType(StoreType.valueOf(type)) - .addAllSubscriptions(subscriptionProtos); - switch (StoreType.valueOf(type)) { - case REDIS: - RedisConfig redisConfig = RedisConfig.parseFrom(config); - return storeProtoBuilder.setRedisConfig(redisConfig).build(); - case REDIS_CLUSTER: - RedisClusterConfig redisClusterConfig = RedisClusterConfig.parseFrom(config); - return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig).build(); - default: - throw new InvalidProtocolBufferException("Invalid store set"); - } - } - - /** - * Returns a List of Subscriptions. - * - * @return List of Subscription - */ - public List getSubscriptions() { - return Arrays.stream(subscriptions.split(",")) - .map(s -> convertStringToSubscription(s)) - .collect(Collectors.toList()); - } - - @Override - public int hashCode() { - return Objects.hash(this.name, this.type, this.subscriptions) ^ Arrays.hashCode(this.config); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (getClass() != obj.getClass()) return false; - Store other = (Store) obj; - - if (!name.equals(other.name)) { - return false; - } else if (!type.equals(other.type)) { - return false; - } else if (!Arrays.equals(config, config)) { - return false; - } else if (!subscriptions.equals(other.subscriptions)) { - return false; - } - return true; - } -} diff --git a/core/src/main/java/feast/core/service/ProjectService.java b/core/src/main/java/feast/core/service/ProjectService.java deleted file mode 100644 index 308c79bccf6..00000000000 --- a/core/src/main/java/feast/core/service/ProjectService.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import feast.core.dao.ProjectRepository; -import feast.core.model.Project; -import java.util.List; -import java.util.Optional; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -@Slf4j -@Service -public class ProjectService { - - private ProjectRepository projectRepository; - - @Autowired - public ProjectService(ProjectRepository projectRepository) { - this.projectRepository = projectRepository; - } - - /** - * Creates a project - * - * @param name Name of project to be created - */ - @Transactional - public void createProject(String name) { - if (projectRepository.existsById(name)) { - throw new IllegalArgumentException(String.format("Project already exists: %s", name)); - } - Project project = new Project(name); - projectRepository.saveAndFlush(project); - } - - /** - * Archives a project - * - * @param name Name of the project to be archived - */ - @Transactional - public void archiveProject(String name) { - Optional project = projectRepository.findById(name); - if (!project.isPresent()) { - throw new IllegalArgumentException(String.format("Could not find project: \"%s\"", name)); - } - if (name.equals(Project.DEFAULT_NAME)) { - throw new UnsupportedOperationException("Archiving the default project is not allowed."); - } - Project p = project.get(); - p.setArchived(true); - projectRepository.saveAndFlush(p); - } - - /** - * List all active projects - * - * @return List of active projects - */ - @Transactional - public List listProjects() { - return projectRepository.findAllByArchivedIsFalse(); - } -} diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java deleted file mode 100644 index 4a35d3ef3a4..00000000000 --- a/core/src/main/java/feast/core/service/SpecService.java +++ /dev/null @@ -1,490 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static feast.core.validators.Matchers.checkValidCharacters; -import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.dao.EntityRepository; -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.ProjectRepository; -import feast.core.dao.StoreRepository; -import feast.core.exception.RetrievalException; -import feast.core.model.*; -import feast.core.validators.EntityValidator; -import feast.core.validators.FeatureTableValidator; -import feast.proto.core.CoreServiceProto.ApplyEntityResponse; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableRequest; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableResponse; -import feast.proto.core.CoreServiceProto.DeleteFeatureTableRequest; -import feast.proto.core.CoreServiceProto.GetEntityRequest; -import feast.proto.core.CoreServiceProto.GetEntityResponse; -import feast.proto.core.CoreServiceProto.GetFeatureTableRequest; -import feast.proto.core.CoreServiceProto.GetFeatureTableResponse; -import feast.proto.core.CoreServiceProto.ListEntitiesRequest; -import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListFeaturesRequest; -import feast.proto.core.CoreServiceProto.ListFeaturesResponse; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.ListStoresResponse.Builder; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -/** - * Facilitates management of specs within the Feast registry. This includes getting existing specs - * and registering new specs. - */ -@Slf4j -@Service -public class SpecService { - - private final EntityRepository entityRepository; - private final FeatureTableRepository tableRepository; - private final ProjectRepository projectRepository; - private final StoreRepository storeRepository; - - @Autowired - public SpecService( - EntityRepository entityRepository, - FeatureTableRepository tableRepository, - StoreRepository storeRepository, - ProjectRepository projectRepository) { - this.entityRepository = entityRepository; - this.tableRepository = tableRepository; - this.storeRepository = storeRepository; - this.projectRepository = projectRepository; - } - - /** - * Get an entity matching the entity name and set project. The entity name and project are - * required. If the project is omitted, the default would be used. - * - * @param request GetEntityRequest Request - * @return Returns a GetEntityResponse containing an entity - */ - public GetEntityResponse getEntity(GetEntityRequest request) { - String projectName = request.getProject(); - String entityName = request.getName(); - - if (entityName.isEmpty()) { - throw new IllegalArgumentException("No entity name provided"); - } - // Autofill default project if project is not specified - if (projectName.isEmpty()) { - projectName = Project.DEFAULT_NAME; - } - - checkValidCharacters(projectName, "project"); - checkValidCharacters(entityName, "entity"); - - EntityV2 entity = entityRepository.findEntityByNameAndProject_Name(entityName, projectName); - - if (entity == null) { - throw new RetrievalException( - String.format("Entity with name \"%s\" could not be found.", entityName)); - } - - // Build GetEntityResponse - GetEntityResponse response = GetEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - - return response; - } - - /** - * Return a map of feature references and features matching the project, labels and entities - * provided in the filter. All fields are required. - * - *

Project name must be explicitly provided or if the project name is omitted, the default - * project would be used. A combination of asterisks/wildcards and text is not allowed. - * - *

The entities in the filter accepts a list. All matching features will be returned. Regex is - * not supported. If no entities are provided, features will not be filtered by entities. - * - *

The labels in the filter accepts a map. All matching features will be returned. Regex is not - * supported. If no labels are provided, features will not be filtered by labels. - * - * @param filter filter containing the desired project name, entities and labels - * @return ListEntitiesResponse with map of feature references and features found matching the - * filter - */ - public ListFeaturesResponse listFeatures(ListFeaturesRequest.Filter filter) { - String project = filter.getProject(); - List entities = filter.getEntitiesList(); - Map labels = filter.getLabelsMap(); - - checkValidCharactersAllowAsterisk(project, "project"); - - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } - - // Currently defaults to all FeatureTables - List featureTables = tableRepository.findAllByProject_Name(project); - - ListFeaturesResponse.Builder response = ListFeaturesResponse.newBuilder(); - if (entities.size() > 0) { - featureTables = - featureTables.stream() - .filter(featureTable -> featureTable.hasAllEntities(entities)) - .collect(Collectors.toList()); - } - - Map featuresMap; - for (FeatureTable featureTable : featureTables) { - featuresMap = featureTable.getFeaturesByLabels(labels); - for (Map.Entry entry : featuresMap.entrySet()) { - response.putFeatures(entry.getKey(), entry.getValue().toProto()); - } - } - - return response.build(); - } - - /** - * Return a list of entities matching the entity name, project and labels provided in the filter. - * All fields are required. Use '*' in entity name and project, and empty map in labels in order - * to return all entities in all projects. - * - *

Project name can be explicitly provided, or an asterisk can be provided to match all - * projects. It is not possible to provide a combination of asterisks/wildcards and text. If the - * project name is omitted, the default project would be used. - * - *

The entity name in the filter accepts an asterisk as a wildcard. All matching entities will - * be returned. Regex is not supported. Explicitly defining an entity name is not possible if a - * project name is not set explicitly. - * - *

The labels in the filter accepts a map. All entities which contain every provided label will - * be returned. - * - * @param filter Filter containing the desired entity name, project and labels - * @return ListEntitiesResponse with list of entities found matching the filter - */ - public ListEntitiesResponse listEntities(ListEntitiesRequest.Filter filter) { - String project = filter.getProject(); - Map labelsFilter = filter.getLabelsMap(); - - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } - - checkValidCharacters(project, "project"); - - List entities = entityRepository.findAllByProject_Name(project); - - ListEntitiesResponse.Builder response = ListEntitiesResponse.newBuilder(); - if (entities.size() > 0) { - entities = - entities.stream() - .filter(entity -> entity.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - for (EntityV2 entity : entities) { - response.addEntities(entity.toProto()); - } - } - - return response.build(); - } - - /** - * Get stores matching the store name provided in the filter. If the store name is not provided, - * the method will return all stores currently registered to Feast. - * - * @param filter filter containing the desired store name - * @return ListStoresResponse containing list of stores found matching the filter - */ - @Transactional - public ListStoresResponse listStores(ListStoresRequest.Filter filter) { - try { - String name = filter.getName(); - if (name.equals("")) { - Builder responseBuilder = ListStoresResponse.newBuilder(); - for (Store store : storeRepository.findAll()) { - responseBuilder.addStore(store.toProto()); - } - return responseBuilder.build(); - } - Store store = - storeRepository - .findById(name) - .orElseThrow( - () -> - new RetrievalException( - String.format("Store with name '%s' not found", name))); - return ListStoresResponse.newBuilder().addStore(store.toProto()).build(); - } catch (InvalidProtocolBufferException e) { - - throw io.grpc.Status.NOT_FOUND - .withDescription("Unable to retrieve stores") - .withCause(e) - .asRuntimeException(); - } - } - - /** - * Creates or updates an entity in the repository. - * - *

This function is idempotent. If no changes are detected in the incoming entity's schema, - * this method will return the existing entity stored in the repository. If project is not - * specified, the entity will be assigned to the 'default' project. - * - * @param newEntitySpec EntitySpecV2 that will be used to create or update an Entity. - * @param projectName Project namespace of Entity which is to be created/updated - */ - @Transactional - public ApplyEntityResponse applyEntity( - EntityProto.EntitySpecV2 newEntitySpec, String projectName) { - // Autofill default project if not specified - if (projectName == null || projectName.isEmpty()) { - projectName = Project.DEFAULT_NAME; - } - - // Validate incoming entity - EntityValidator.validateSpec(newEntitySpec); - - // Find project or create new one if it does not exist - Project project = projectRepository.findById(projectName).orElse(new Project(projectName)); - - // Ensure that the project retrieved from repository is not archived - if (project.isArchived()) { - throw new IllegalArgumentException(String.format("Project is archived: %s", projectName)); - } - - // Retrieve existing Entity - EntityV2 entity = - entityRepository.findEntityByNameAndProject_Name(newEntitySpec.getName(), projectName); - - EntityProto.Entity newEntity = EntityProto.Entity.newBuilder().setSpec(newEntitySpec).build(); - if (entity == null) { - // Create new entity since it doesn't exist - entity = EntityV2.fromProto(newEntity); - } else { - // If the entity remains unchanged, we do nothing. - if (entity.toProto().getSpec().equals(newEntitySpec)) { - return ApplyEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - } - entity.updateFromProto(newEntity, projectName); - } - - // Persist the EntityV2 object - project.addEntity(entity); - projectRepository.saveAndFlush(project); - - // Build ApplyEntityResponse - ApplyEntityResponse response = - ApplyEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - return response; - } - - /** - * Resolves the project name by returning name if given, autofilling default project otherwise. - * - * @param projectName name of the project to resolve. - */ - public static String resolveProjectName(String projectName) { - return (projectName.isEmpty()) ? Project.DEFAULT_NAME : projectName; - } - - /** - * UpdateStore updates the repository with the new given store. - * - * @param updateStoreRequest containing the new store definition - * @return UpdateStoreResponse containing the new store definition - */ - @Transactional - public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) - throws InvalidProtocolBufferException { - StoreProto.Store newStoreProto = updateStoreRequest.getStore(); - - List subs = newStoreProto.getSubscriptionsList(); - for (Subscription sub : subs) { - // Ensure that all fields in a subscription contain values - if ((sub.getName().isEmpty()) || sub.getProject().isEmpty()) { - throw new IllegalArgumentException( - String.format("Missing parameter in subscription: %s", sub)); - } - } - Store existingStore = storeRepository.findById(newStoreProto.getName()).orElse(null); - - // Do nothing if no change - if (existingStore != null && existingStore.toProto().equals(newStoreProto)) { - return UpdateStoreResponse.newBuilder() - .setStatus(UpdateStoreResponse.Status.NO_CHANGE) - .setStore(updateStoreRequest.getStore()) - .build(); - } - - Store newStore = Store.fromProto(newStoreProto); - storeRepository.save(newStore); - return UpdateStoreResponse.newBuilder() - .setStatus(UpdateStoreResponse.Status.UPDATED) - .setStore(updateStoreRequest.getStore()) - .build(); - } - - /** - * Applies the given FeatureTable to the FeatureTable registry. Creates the FeatureTable if does - * not exist, otherwise updates the existing FeatureTable. Applies FeatureTable in project if - * specified, otherwise in default project. - * - * @param request Contains FeatureTable spec and project parameters used to create or update a - * FeatureTable. - * @throws NoSuchElementException projects and entities referenced in request do not exist. - * @return response containing the applied FeatureTable spec. - */ - @Transactional - public ApplyFeatureTableResponse applyFeatureTable(ApplyFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - - // Check that specification provided is valid - FeatureTableSpec applySpec = request.getTableSpec(); - FeatureTableValidator.validateSpec(applySpec); - - // Prevent apply if the project is archived. - Project project = projectRepository.findById(projectName).orElse(new Project(projectName)); - if (project.isArchived()) { - throw new IllegalArgumentException( - String.format( - "Cannot apply Feature Table to archived Project: (table: %s, project: %s)", - applySpec.getName(), projectName)); - } - - // Create or update depending on whether there is an existing Feature Table - Optional existingTable = - tableRepository.findFeatureTableByNameAndProject_Name(applySpec.getName(), projectName); - FeatureTable table = FeatureTable.fromProto(projectName, applySpec, entityRepository); - if (existingTable.isPresent() && table.equals(existingTable.get())) { - // Skip update if no change is detected - return ApplyFeatureTableResponse.newBuilder().setTable(existingTable.get().toProto()).build(); - } - if (existingTable.isPresent()) { - existingTable.get().updateFromProto(projectName, applySpec, entityRepository); - table = existingTable.get(); - } - - // Commit FeatureTable to database and return applied FeatureTable - tableRepository.saveAndFlush(table); - return ApplyFeatureTableResponse.newBuilder().setTable(table.toProto()).build(); - } - - /** - * List the FeatureTables matching the filter in the given filter. Scopes down listing to project - * if specified, the default project otherwise. - * - * @param filter Filter containing the desired project and labels - * @return ListFeatureTablesResponse with list of FeatureTables found matching the filter - */ - @Transactional - public ListFeatureTablesResponse listFeatureTables(ListFeatureTablesRequest.Filter filter) { - String projectName = resolveProjectName(filter.getProject()); - Map labelsFilter = filter.getLabelsMap(); - - checkValidCharacters(projectName, "project"); - - List matchingTables = tableRepository.findAllByProject_Name(projectName); - - ListFeatureTablesResponse.Builder response = ListFeatureTablesResponse.newBuilder(); - - if (matchingTables.size() > 0) { - matchingTables = - matchingTables.stream() - .filter(table -> table.hasAllLabels(labelsFilter)) - .filter(table -> !table.isDeleted()) - .collect(Collectors.toList()); - } - for (FeatureTable table : matchingTables) { - response.addTables(table.toProto()); - } - - return response.build(); - } - - /** - * Get the FeatureTable with the name and project specified in the request. Gets FeatureTable in - * project if specified, otherwise in default project. - * - * @param request containing the retrieval parameters. - * @throws NoSuchElementException if no FeatureTable matches given request. - * @return response containing the requested FeatureTable. - */ - @Transactional - public GetFeatureTableResponse getFeatureTable(GetFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - String featureTableName = request.getName(); - - checkValidCharacters(projectName, "project"); - checkValidCharacters(featureTableName, "featureTable"); - - Optional retrieveTable = - tableRepository.findFeatureTableByNameAndProject_Name(featureTableName, projectName); - if (retrieveTable.isEmpty()) { - throw new NoSuchElementException( - String.format( - "No such Feature Table: (project: %s, name: %s)", projectName, featureTableName)); - } - - if (retrieveTable.get().isDeleted()) { - throw new NoSuchElementException( - String.format( - "Feature Table has been deleted: (project: %s, name: %s)", - projectName, featureTableName)); - } - - // Build GetFeatureTableResponse - GetFeatureTableResponse response = - GetFeatureTableResponse.newBuilder().setTable(retrieveTable.get().toProto()).build(); - - return response; - } - - @Transactional - public void deleteFeatureTable(DeleteFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - String featureTableName = request.getName(); - - checkValidCharacters(projectName, "project"); - checkValidCharacters(featureTableName, "featureTable"); - - Optional existingTable = - tableRepository.findFeatureTableByNameAndProject_Name(featureTableName, projectName); - if (existingTable.isEmpty()) { - throw new NoSuchElementException( - String.format( - "No such Feature Table: (project: %s, name: %s)", projectName, featureTableName)); - } - - existingTable.get().delete(); - } -} diff --git a/core/src/main/java/feast/core/util/StreamUtil.java b/core/src/main/java/feast/core/util/StreamUtil.java deleted file mode 100644 index 4925f023100..00000000000 --- a/core/src/main/java/feast/core/util/StreamUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import java.util.function.Function; - -/** Collection of functions useful for stream-style programming */ -public class StreamUtil { - - @FunctionalInterface - public interface CheckedFunction { - R apply(T t) throws Exception; - } - - /** - * Wrap function to convert its checked exceptions into RuntimeException - * - * @param checkedFunction function that throws checked exception - * @param input - * @param output - * @return wrapped function that doesn't throw checked exceptions - */ - public static Function wrapException(CheckedFunction checkedFunction) { - return t -> { - try { - return checkedFunction.apply(t); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - } -} diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java deleted file mode 100644 index bbdfa948049..00000000000 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import feast.proto.types.ValueProto.ValueType.Enum; -import java.lang.reflect.Type; -import java.util.*; - -public class TypeConversion { - private static Gson gson = new Gson(); - - /** - * Convert a java data object to protobuf Timestamp object - * - * @param ts timestamp - * @return protobuf.Timestamp object of the given timestamp - */ - public static com.google.protobuf.Timestamp convertTimestamp(Date ts) { - return com.google.protobuf.Timestamp.newBuilder().setSeconds(ts.getTime() / 1000).build(); - } - - /** - * Convert a string of comma-separated strings to list of strings - * - * @param tags comma separated tags - * @return list of tags - */ - public static List convertTagStringToList(String tags) { - if (tags == null || tags.isEmpty()) { - return Collections.emptyList(); - } - return Arrays.asList(tags.split(",")); - } - - /** - * Unmarshals a given json string to map - * - * @param jsonString valid json formatted string - * @return map of keys to values in json - */ - public static Map convertJsonStringToMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } - - /** - * Unmarshals a given json string to Enum map - * - * @param jsonString valid json formatted string - * @return map of keys to Enum values in json string - */ - public static Map convertJsonStringToEnumMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } - - /** - * Marshals a given map into its corresponding json string - * - * @param map - * @return json string corresponding to given map - */ - public static String convertMapToJsonString(Map map) { - return gson.toJson(map); - } - - /** - * Marshals a given Enum map into its corresponding json string - * - * @param map - * @return json string corresponding to given Enum map - */ - public static String convertEnumMapToJsonString(Map map) { - return gson.toJson(map); - } -} diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java deleted file mode 100644 index f36e3609d7e..00000000000 --- a/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.*; -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; - -import feast.proto.core.DataFormatProto.FileFormat; -import feast.proto.core.DataFormatProto.StreamFormat; -import feast.proto.core.DataSourceProto.DataSource; - -public class DataSourceValidator { - /** Validate if the given DataSource protobuf spec is valid. */ - public static void validate(DataSource spec) { - switch (spec.getType()) { - case BATCH_FILE: - FileFormat.FormatCase fileFormat = spec.getFileOptions().getFileFormat().getFormatCase(); - switch (fileFormat) { - case PARQUET_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported File Format: %s", fileFormat)); - } - break; - - case BATCH_BIGQUERY: - checkValidBigQueryTableRef(spec.getBigqueryOptions().getTableRef(), "FeatureTable"); - break; - - case STREAM_KAFKA: - StreamFormat.FormatCase messageFormat = - spec.getKafkaOptions().getMessageFormat().getFormatCase(); - switch (messageFormat) { - case PROTO_FORMAT: - checkValidClassPath( - spec.getKafkaOptions().getMessageFormat().getProtoFormat().getClassPath(), - "FeatureTable"); - break; - case AVRO_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format( - "Unsupported Stream Format for Kafka Source Type: %s", messageFormat)); - } - break; - - case STREAM_KINESIS: - // Verify tht DataFormat is supported by kinesis data source - StreamFormat.FormatCase recordFormat = - spec.getKinesisOptions().getRecordFormat().getFormatCase(); - switch (recordFormat) { - case PROTO_FORMAT: - checkValidClassPath( - spec.getKinesisOptions().getRecordFormat().getProtoFormat().getClassPath(), - "FeatureTable"); - break; - case AVRO_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Stream Format for Kafka Source Type: %s", recordFormat)); - } - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); - } - } -} diff --git a/core/src/main/java/feast/core/validators/EntityValidator.java b/core/src/main/java/feast/core/validators/EntityValidator.java deleted file mode 100644 index 743a0446902..00000000000 --- a/core/src/main/java/feast/core/validators/EntityValidator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.checkValidCharacters; - -import feast.proto.core.EntityProto; - -public class EntityValidator { - public static void validateSpec(EntityProto.EntitySpecV2 entitySpec) { - if (entitySpec.getName().isEmpty()) { - throw new IllegalArgumentException("Entity name must be provided"); - } - if (entitySpec.getValueType().toString().isEmpty()) { - throw new IllegalArgumentException("Entity type must not be empty"); - } - if (entitySpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Entity label keys must not be empty"); - } - - checkValidCharacters(entitySpec.getName(), "entity"); - } -} diff --git a/core/src/main/java/feast/core/validators/FeatureTableValidator.java b/core/src/main/java/feast/core/validators/FeatureTableValidator.java deleted file mode 100644 index 863c9442f77..00000000000 --- a/core/src/main/java/feast/core/validators/FeatureTableValidator.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.*; - -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Set; -import java.util.stream.Collectors; - -public class FeatureTableValidator { - protected static final Set RESERVED_NAMES = - Set.of("created_timestamp", "event_timestamp"); - - public static void validateSpec(FeatureTableSpec spec) { - if (spec.getName().isEmpty()) { - throw new IllegalArgumentException("FeatureTable name must be provided"); - } - if (spec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("FeatureTable cannot have labels with empty key."); - } - if (spec.getEntitiesCount() == 0) { - throw new IllegalArgumentException("FeatureTable entities list cannot be empty."); - } - if (spec.getFeaturesCount() == 0) { - throw new IllegalArgumentException("FeatureTable features list cannot be empty."); - } - if (!spec.hasBatchSource()) { - throw new IllegalArgumentException("FeatureTable batch source cannot be empty."); - } - - checkValidCharacters(spec.getName(), "FeatureTable"); - spec.getFeaturesList().forEach(FeatureTableValidator::validateFeatureSpec); - - // Check that features and entities defined in FeatureTable do not use reserved names - ArrayList fieldNames = new ArrayList<>(spec.getEntitiesList()); - fieldNames.addAll( - spec.getFeaturesList().stream().map(FeatureSpecV2::getName).collect(Collectors.toList())); - if (!Collections.disjoint(fieldNames, RESERVED_NAMES)) { - throw new IllegalArgumentException( - String.format( - "Reserved names has been used as Feature(s) names. Reserved: %s", RESERVED_NAMES)); - } - - // Check that Feature and Entity names in FeatureTable do not collide with each other - if (hasDuplicates(fieldNames)) { - throw new IllegalArgumentException( - String.format("Entity and Feature names within a Feature Table should be unique.")); - } - - // Check that the data sources defined in the feature table are valid - if (!spec.getBatchSource().getType().equals(SourceType.INVALID)) { - DataSourceValidator.validate(spec.getBatchSource()); - } - if (!spec.getStreamSource().getType().equals(SourceType.INVALID)) { - DataSourceValidator.validate(spec.getStreamSource()); - } - } - - private static void validateFeatureSpec(FeatureSpecV2 spec) { - checkValidCharacters(spec.getName(), "Feature"); - if (spec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Features cannot have labels with empty key."); - } - } -} diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java deleted file mode 100644 index 5f7ddd26ac5..00000000000 --- a/core/src/main/java/feast/core/validators/Matchers.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import java.util.Collection; -import java.util.HashSet; -import java.util.regex.Pattern; - -public class Matchers { - - private static Pattern BIGQUERY_TABLE_REF_REGEX = - Pattern.compile("[a-zA-Z0-9-]+[:]+[a-zA-Z0-9_]+[.]+[a-zA-Z0-9_]*"); - private static Pattern CLASS_PATH_REGEX = - Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*(\\.[a-zA-Z_][a-zA-Z0-9_]*)*$"); - private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); - private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); - private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); - private static Pattern VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD = - Pattern.compile("^[a-zA-Z0-9\\-_*]*$"); - - private static String ERROR_MESSAGE_TEMPLATE = "invalid value for %s resource, %s: %s"; - - public static void checkUpperSnakeCase(String input, String resource) - throws IllegalArgumentException { - if (!UPPER_SNAKE_CASE_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in upper snake case, and cannot include any special characters.")); - } - } - - public static void checkLowerSnakeCase(String input, String resource) - throws IllegalArgumentException { - if (!LOWER_SNAKE_CASE_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in lower snake case, and cannot include any special characters.")); - } - } - - public static void checkValidCharacters(String input, String resource) - throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must only contain alphanumeric characters and underscores.")); - } - } - - public static void checkValidCharactersAllowAsterisk(String input, String resource) - throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX_WITH_ASTERISK_WILDCARD.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must only contain alphanumeric characters, dashes, underscores, or an asterisk.")); - } - } - - public static void checkValidBigQueryTableRef(String input, String resource) - throws IllegalArgumentException { - if (!BIGQUERY_TABLE_REF_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in the form of .")); - } - } - - public static void checkValidClassPath(String input, String resource) { - if (!CLASS_PATH_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, resource, input, "argument must be a valid Java Classpath")); - } - } - - public static boolean hasDuplicates(Collection strings) { - return (new HashSet<>(strings)).size() < strings.size(); - } -} diff --git a/core/src/main/resources/application.yml b/core/src/main/resources/application.yml deleted file mode 100644 index 1c7b14f415d..00000000000 --- a/core/src/main/resources/application.yml +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -feast: - stream: - # Feature stream type. Only kafka is supported. - type: kafka - # Feature stream options. - # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig - options: - topic: feast-features - bootstrapServers: localhost:9092 - replicationFactor: 1 - partitions: 1 - specsOptions: - specsTopic: feast-specs - specsAckTopic: feast-specs-ack - notifyIntervalMilliseconds: 1000 - - security: - authentication: - enabled: false - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - subjectClaim: email - - authorization: - enabled: false - provider: http - options: - authorizationUrl: http://localhost:8082 - - # If set to true, HTTP REST endpoints at /api/v1 implemented by - # CoreServiceRestController will be accessible in Feast Core WITHOUT - # authentication. - disableRestControllerAuth: false - - logging: - # Audit logging provides a machine readable structured JSON log that can give better - # insight into what is happening in Feast. - audit: - # Whether audit logging is enabled. - enabled: true - # Whether to enable message level (ie request/response) audit logging - messageLogging: - enabled: false - # Logging forwarder currently provides a machine readable structured JSON log to an - # external fluentd service that can give better insight into what is happening in Feast. - # Accepts console / fluentd as destination - destination: console - fluentdHost: localhost - fluentdPort: 24224 - -grpc: - server: - # The port that Feast Core gRPC service listens on - port: 6565 - security: - enabled: false - certificateChain: server.crt - privateKey: server.key - -spring: - jpa: - properties.hibernate: - format_sql: true - event: - merge: - entity_copy_observer: allow - hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl - hibernate.ddl-auto: validate - datasource: - driverClassName: org.postgresql.Driver - url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres} - username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} - flyway: - baseline-on-migrate: true - -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: true - host: ${STATSD_HOST:localhost} - port: ${STATSD_PORT:8125} diff --git a/core/src/main/resources/banner.txt b/core/src/main/resources/banner.txt deleted file mode 100644 index d0f1c033ebe..00000000000 --- a/core/src/main/resources/banner.txt +++ /dev/null @@ -1,14 +0,0 @@ - -███████╗███████╗ █████╗ ███████╗████████╗ -██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝ -█████╗ █████╗ ███████║███████╗ ██║ -██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║ -██║ ███████╗██║ ██║███████║ ██║ -╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ - - ██████╗ ██████╗ ██████╗ ███████╗ -██╔════╝██╔═══██╗██╔══██╗██╔════╝ -██║ ██║ ██║██████╔╝█████╗ -██║ ██║ ██║██╔══██╗██╔══╝ -╚██████╗╚██████╔╝██║ ██║███████╗ - ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝ diff --git a/core/src/main/resources/db/migration/V1__Baseline.sql b/core/src/main/resources/db/migration/V1__Baseline.sql deleted file mode 100644 index 262e0586ef2..00000000000 --- a/core/src/main/resources/db/migration/V1__Baseline.sql +++ /dev/null @@ -1,185 +0,0 @@ --- --- Dump of Feast Database (as of RELEASE 0.5) --- Baseline dump for migrating to flyway --- - - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET default_tablespace = ''; - - -CREATE TABLE entities ( - id bigint NOT NULL, - name character varying(255), - type character varying(255), - feature_set_id bigint -); - - -CREATE TABLE feature_sets ( - id bigint NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - labels text, - max_age bigint, - name character varying(255) NOT NULL, - status character varying(255), - project_name character varying(255), - source character varying(255) -); - - -CREATE TABLE features ( - id bigint NOT NULL, - archived boolean NOT NULL, - bool_domain bytea, - domain character varying(255), - float_domain bytea, - group_presence bytea, - image_domain bytea, - int_domain bytea, - labels text, - mid_domain bytea, - name character varying(255), - natural_language_domain bytea, - presence bytea, - shape bytea, - string_domain bytea, - struct_domain bytea, - time_domain bytea, - time_of_day_domain bytea, - type character varying(255), - url_domain bytea, - value_count bytea, - feature_set_id bigint -); - - - -CREATE SEQUENCE hibernate_sequence - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -CREATE TABLE jobs ( - id character varying(255) NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - ext_id character varying(255), - runner character varying(255), - status character varying(16), - source_id character varying(255), - store_name character varying(255) -); - - -CREATE TABLE jobs_feature_sets ( - job_id character varying(255) NOT NULL, - feature_sets_id bigint NOT NULL -); - - -CREATE TABLE projects ( - name character varying(255) NOT NULL, - archived boolean NOT NULL -); - - -CREATE TABLE sources ( - id character varying(255) NOT NULL, - bootstrap_servers character varying(255), - is_default boolean, - topics character varying(255), - type character varying(255) NOT NULL -); - - -CREATE TABLE stores ( - name character varying(255) NOT NULL, - config oid NOT NULL, - subscriptions character varying(255), - type character varying(255) NOT NULL -); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT entities_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT feature_sets_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY features - ADD CONSTRAINT features_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT jobs_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY projects - ADD CONSTRAINT projects_pkey PRIMARY KEY (name); - - -ALTER TABLE ONLY sources - ADD CONSTRAINT sources_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY stores - ADD CONSTRAINT stores_pkey PRIMARY KEY (name); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT uk4hredqqfh86prhp1hf08nofvk UNIQUE (name, feature_set_id); - - -ALTER TABLE ONLY features - ADD CONSTRAINT ukedouxmpcoev743cmstfwq25yp UNIQUE (name, feature_set_id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT ukoajkc7tn9nwhodjrbcjri5jix UNIQUE (name, project_name); - - -CREATE INDEX idx_jobs_feature_sets_feature_sets_id ON jobs_feature_sets USING btree (feature_sets_id); - -CREATE INDEX idx_jobs_feature_sets_job_id ON jobs_feature_sets USING btree (job_id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh FOREIGN KEY (source) REFERENCES sources(id); - - -ALTER TABLE ONLY jobs_feature_sets - ADD CONSTRAINT fk2qt5yj45cr02spdhp59h4wpeg FOREIGN KEY (job_id) REFERENCES jobs(id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT fk3dwuno3phk8j3iwdl4cckdqqd FOREIGN KEY (store_name) REFERENCES stores(name); - - -ALTER TABLE ONLY features - ADD CONSTRAINT fkfxcpsscvj0g89o4p5dx4insb1 FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f FOREIGN KEY (source_id) REFERENCES sources(id); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT fkhyblh5sfunv00a8ums8ms9otq FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT fkiiqcdeuuq9mf0tmt7jtnln3oa FOREIGN KEY (project_name) REFERENCES projects(name); - - -ALTER TABLE ONLY jobs_feature_sets - ADD CONSTRAINT fkroca9etjw89c48e8jays6jl4l FOREIGN KEY (feature_sets_id) REFERENCES feature_sets(id); diff --git a/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql b/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql deleted file mode 100644 index c70df670391..00000000000 --- a/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Migrating to Many2Many relationship between Job and Store - -CREATE TABLE jobs_stores( - job_id character varying(255) NOT NULL, - store_name character varying(255) NOT NULL -); - -ALTER TABLE ONLY jobs_stores - ADD CONSTRAINT jobs_stores_job_fkey FOREIGN KEY (job_id) REFERENCES jobs(id); - -ALTER TABLE ONLY jobs_stores - ADD CONSTRAINT jobs_stores_store_fkey FOREIGN KEY (store_name) REFERENCES stores(name); - - -CREATE INDEX idx_jobs_stores_job_id ON jobs_stores USING btree (job_id); -CREATE INDEX idx_jobs_stores_store_name ON jobs_stores USING btree (store_name); diff --git a/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql b/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql deleted file mode 100644 index 624bf773770..00000000000 --- a/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql +++ /dev/null @@ -1,3 +0,0 @@ -UPDATE stores - SET subscriptions = array_to_string(string_to_array(subscriptions, ':') || '{false}', ':') - WHERE array_length(string_to_array(subscriptions, ':'), 1) = 2; \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql b/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql deleted file mode 100644 index 98cb8c2aaa5..00000000000 --- a/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE jobs_feature_sets ADD CONSTRAINT jobs_feature_sets_pkey PRIMARY KEY (job_id, feature_sets_id); - -ALTER TABLE jobs_stores ADD CONSTRAINT jobs_stores_pkey PRIMARY KEY (job_id, store_name); \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2.4__Store_proto.sql b/core/src/main/resources/db/migration/V2.4__Store_proto.sql deleted file mode 100644 index 9e18313e95a..00000000000 --- a/core/src/main/resources/db/migration/V2.4__Store_proto.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE jobs_stores ADD COLUMN store_proto oid not null; \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql b/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql deleted file mode 100644 index 4ef3013c2c7..00000000000 --- a/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql +++ /dev/null @@ -1,14 +0,0 @@ -WITH updates as ( - select name, - array_to_string(array_agg(b.array_to_string), ',') as subscriptions - from ( - select name, array_to_string(string_to_array(subscriptions, ':') || '{false}', ':') - from ( - select name, unnest(string_to_array(subscriptions, ',')) as subscriptions from stores) a - WHERE array_length(string_to_array(subscriptions, ':'), 1) = 2) b - group by name -) -UPDATE stores -SET subscriptions = updates.subscriptions -FROM updates -WHERE stores.name = updates.name \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql b/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql deleted file mode 100644 index 0d824aed70d..00000000000 --- a/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql +++ /dev/null @@ -1,7 +0,0 @@ --- SQL migration to migrate create 'default' project if it does not already exist. -INSERT INTO projects - (name, archived) - SELECT 'default', false - WHERE NOT EXISTS ( - SELECT name FROM projects WHERE name='default' - ); diff --git a/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql b/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql deleted file mode 100644 index 183664ad658..00000000000 --- a/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Migrating to Entities as a higher-level concept - -CREATE TABLE entities_v2( - id bigint NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - name character varying(255), - project_name character varying(255), - type character varying(255), - description text, - labels text -); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_project_ukey UNIQUE (name, project_name); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_project_fkey FOREIGN KEY (project_name) REFERENCES projects(name); \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql b/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql deleted file mode 100644 index a083dfd1ae6..00000000000 --- a/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql +++ /dev/null @@ -1,66 +0,0 @@ --- Data Sources SQL table used to Store Feature project -CREATE TABLE data_sources ( - id bigint NOT NULL, - type character varying(255) NOT NULL, - field_mapping text NOT NULL, - timestamp_column character varying(255), - date_partition_column character varying(255), - -- Only the options corresponding to type should set & non-null - -- DataSource Options - config text, - - CONSTRAINT data_sources_pkey PRIMARY KEY (id) -); - --- Feature Table SQL table used to store FeatureTables protobuf -CREATE TABLE feature_tables ( - id bigint NOT NULL, - project_name character varying(255), - name character varying(255) NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - labels text NOT NULL, - max_age bigint NOT NULL, - stream_source_id bigint, - batch_source_id bigint, - revision int NOT NULL , - - CONSTRAINT feature_tables_pkey PRIMARY KEY (id), - CONSTRAINT feature_tables_project_fkey FOREIGN KEY (project_name) - REFERENCES projects(name), - CONSTRAINT feature_tables_stream_data_source_fkey FOREIGN KEY (stream_source_id) - REFERENCES data_sources(id), - CONSTRAINT feature_tables_batch_data_source_fkey FOREIGN KEY (batch_source_id) - REFERENCES data_sources(id), - -- Feature Tables must be unique within a project - CONSTRAINT feature_tables_unique_project UNIQUE (name, project_name) -); - --- Join table between feature tables and entities V2 -CREATE TABLE feature_tables_entities_v2 ( - feature_table_id bigint NOT NULL, - entity_v2_id bigint NOT NULL, - - CONSTRAINT feature_tables_entities_v2_pkey PRIMARY KEY (feature_table_id, entity_v2_id), - CONSTRAINT feature_tables_entities_v2_join_feature_tables_fkey - FOREIGN KEY (feature_table_id) REFERENCES feature_tables(id), - CONSTRAINT feature_tables_entities_v2_join_entities_v2_fkey - FOREIGN KEY (entity_v2_id) REFERENCES entities_v2 (id) -); - - --- Feature v2 SQL table used to store FeatureSpecV2 protobuf -CREATE TABLE features_v2 ( - id bigint NOT NULL, - feature_table_id bigint NOT NULL, - name character varying(255), - type character varying(255), - labels text NOT NULL, - - CONSTRAINT features_v2_pkey PRIMARY KEY (id), - CONSTRAINT features_v2_feature_table_fkey FOREIGN KEY (feature_table_id) - REFERENCES feature_tables(id), - -- Features should be unique within a feature set - CONSTRAINT feature_v2_feature_table_unique UNIQUE (name, feature_table_id) -); - diff --git a/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql b/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql deleted file mode 100644 index ddbeffc5848..00000000000 --- a/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE data_sources ADD COLUMN created_timestamp_column character varying(255); \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql b/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql deleted file mode 100644 index 230a5f5c8c1..00000000000 --- a/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql +++ /dev/null @@ -1,124 +0,0 @@ ---- Feast Release 0.6 - ---- New fields from FeatureSetJobStatus (version & deliveryStatus) - -ALTER TABLE jobs_feature_sets - ADD column version int4 default 0; - -ALTER TABLE jobs_feature_sets - ADD column delivery_status varchar(255); - -ALTER TABLE feature_sets - ADD column version int4 default 0; - -UPDATE feature_sets SET version = 1; - - ---- FeatureStatistics Creation - -CREATE TABLE feature_statistics -( - id integer NOT NULL, - average_length real NOT NULL, - avg_bytes real NOT NULL, - avg_num_values real NOT NULL, - count bigint NOT NULL, - dataset_id character varying(255), - date timestamp without time zone, - feature_type character varying(255), - max double precision NOT NULL, - max_bytes real NOT NULL, - max_num_values bigint NOT NULL, - mean double precision NOT NULL, - median double precision NOT NULL, - min double precision NOT NULL, - min_bytes real NOT NULL, - min_num_values bigint NOT NULL, - num_missing bigint NOT NULL, - num_values_histogram bytea, - numeric_value_histogram bytea, - numeric_value_quantiles bytea, - rank_histogram bytea, - stdev double precision NOT NULL, - top_values bytea, - total_num_values bigint NOT NULL, - n_unique bigint, - zeroes bigint NOT NULL, - feature_id bigint -); - - -ALTER TABLE ONLY feature_statistics - ADD CONSTRAINT feature_statistics_pkey PRIMARY KEY (id); - -CREATE INDEX idx_feature_statistics_dataset_id ON public.feature_statistics USING btree (dataset_id); - -CREATE INDEX idx_feature_statistics_date ON public.feature_statistics USING btree (date); - -CREATE INDEX idx_feature_statistics_feature ON public.feature_statistics USING btree (feature_id); - -ALTER TABLE ONLY feature_statistics - ADD CONSTRAINT feature_statistics_feature_fkey FOREIGN KEY (feature_id) REFERENCES public.features (id); - - ---- Releasing previous PK in Source - -ALTER TABLE feature_sets - DROP CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh; -ALTER TABLE jobs - DROP CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f; -ALTER TABLE sources - DROP CONSTRAINT sources_pkey; -ALTER TABLE sources - ALTER COLUMN id DROP NOT NULL; - ---- Migrating to auto-incremental Source primary key - -ALTER TABLE sources - ADD column pk SERIAL PRIMARY KEY; - --- ALTER TABLE sources --- ALTER column type Type int4 USING ('{"KAFKA": 1}'::json ->> type)::INTEGER; -ALTER TABLE sources - ADD column config varchar(255); - - ---- Update all related to Source tables - -ALTER TABLE feature_sets - ADD COLUMN source_id int4; - --- foreign key on source(id) -> source(pk) - -ALTER TABLE feature_sets - ADD CONSTRAINT feature_sets_to_sources_fkey - FOREIGN KEY (source_id) REFERENCES sources (pk); - - -ALTER TABLE jobs - ADD COLUMN source_type varchar(255); -- Enum SourceType -ALTER TABLE jobs - ADD COLUMN source_config varchar(255); - - - ---- Migrate Data ---- creating sources with identical primary keys as in feature_sets - -SELECT feature_sets.id, - sources.bootstrap_servers, - sources.topics, - sources.is_default, - sources.type -INTO TEMP TABLE converted_sources -FROM feature_sets, - sources -WHERE feature_sets.source = sources.id; - -DELETE FROM sources; - -INSERT INTO sources (pk, bootstrap_servers, topics, is_default, type) - (SELECT * FROM converted_sources); - -UPDATE feature_sets -SET source_id = id; diff --git a/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql b/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql deleted file mode 100644 index e915ec07d4a..00000000000 --- a/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL; \ No newline at end of file diff --git a/core/src/main/resources/log4j2.xml b/core/src/main/resources/log4j2.xml deleted file mode 100644 index 8781d668a84..00000000000 --- a/core/src/main/resources/log4j2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - diff --git a/core/src/main/resources/templates/bq_training.tmpl b/core/src/main/resources/templates/bq_training.tmpl deleted file mode 100644 index a92666e91bd..00000000000 --- a/core/src/main/resources/templates/bq_training.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -SELECT - id, - event_timestamp{%- if features | length > 0 %},{%- endif %} - {{ features | join(',') }} -FROM - `{{ table_id }}` -WHERE event_timestamp >= TIMESTAMP("{{ start_date }}") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("{{ end_date }}", INTERVAL 1 DAY)) -{%- for key, val in number_filters.items() %} AND {{ key }} = {{ val }} {%- endfor %} -{%- for key, val in string_filters.items() %} AND {{ key }} = "{{ val }}" {%- endfor %} -{% if limit is not none -%} -LIMIT {{ limit }} -{%- endif %} \ No newline at end of file diff --git a/core/src/main/resources/templates/bq_view.tmpl b/core/src/main/resources/templates/bq_view.tmpl deleted file mode 100644 index 861e5c16582..00000000000 --- a/core/src/main/resources/templates/bq_view.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -WITH {{tableName}} AS ( - SELECT - ROW_NUMBER() OVER (PARTITION BY id, event_timestamp ORDER BY created_timestamp ASC) rownum, - id, - event_timestamp, - created_timestamp - {{#features}} - ,FIRST_VALUE({{name}} IGNORE NULLS) OVER w as {{name}} - {{/features}} - FROM - `{{project}}.{{dataset}}.{{tableName}}` - WINDOW w AS ( - PARTITION BY id, event_timestamp ORDER BY event_timestamp DESC - ) -) -SELECT - id, - event_timestamp, - created_timestamp - {{#features}} - ,{{name}} - {{/features}} -FROM {{tableName}} WHERE rownum = 1 \ No newline at end of file diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java deleted file mode 100644 index 7a35fc4369a..00000000000 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core; - -public class CoreApplicationTest {} diff --git a/core/src/test/java/feast/core/annotation/IntegrationTest.java b/core/src/test/java/feast/core/annotation/IntegrationTest.java deleted file mode 100644 index f1617e32a67..00000000000 --- a/core/src/test/java/feast/core/annotation/IntegrationTest.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.annotation; - -public interface IntegrationTest {} diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java deleted file mode 100644 index 4bfa084e155..00000000000 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.service.AuthorizationService; -import feast.common.it.DataGenerator; -import feast.core.config.FeastProperties; -import feast.core.dao.ProjectRepository; -import feast.core.grpc.CoreServiceImpl; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceProto.ApplyEntityRequest; -import feast.proto.core.CoreServiceProto.ApplyEntityResponse; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.internal.testing.StreamRecorder; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; - -public class CoreServiceAuthTest { - - private CoreServiceImpl coreService; - private ProjectService projectService; - - @Mock private SpecService specService; - @Mock private ProjectRepository projectRepository; - @Mock private AuthorizationProvider authProvider; - - public CoreServiceAuthTest() { - MockitoAnnotations.initMocks(this); - SecurityProperties.AuthorizationProperties authProp = - new SecurityProperties.AuthorizationProperties(); - authProp.setEnabled(true); - SecurityProperties sp = new SecurityProperties(); - sp.setAuthorization(authProp); - FeastProperties feastProperties = new FeastProperties(); - feastProperties.setSecurity(sp); - projectService = new ProjectService(projectRepository); - AuthorizationService authService = - new AuthorizationService(feastProperties.getSecurity(), authProvider); - coreService = new CoreServiceImpl(specService, projectService, feastProperties, authService); - } - - @Test - public void shouldNotApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - - String project = "project1"; - Authentication auth = mock(Authentication.class); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(auth); - - doReturn(AuthorizationResult.failed(null)) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - - StreamRecorder responseObserver = StreamRecorder.create(); - EntityProto.EntitySpecV2 incomingEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - ApplyEntityRequest request = - ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - - coreService.applyEntity(request, responseObserver); - assertEquals("PERMISSION_DENIED: Access Denied", responseObserver.getError().getMessage()); - } - - @Test - public void shouldApplyEntityIfProjectMember() throws InvalidProtocolBufferException { - - String project = "project1"; - Authentication auth = mock(Authentication.class); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(auth); - doReturn(AuthorizationResult.success()) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - - StreamRecorder responseObserver = StreamRecorder.create(); - EntityProto.EntitySpecV2 incomingEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - ApplyEntityRequest request = - ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - - coreService.applyEntity(request, responseObserver); - } -} diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java deleted file mode 100644 index 77cae43cd87..00000000000 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.*; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import java.util.*; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false", - }) -public class CoreServiceAuthenticationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String subjectClaim = "sub"; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - static SimpleCoreClient insecureApiClient; - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @AfterAll - static void tearDown() { - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - /** - * If authentication is enabled but authorization is disabled, users can still connect to Feast - * Core as anonymous users. They are not forced to authenticate. - */ - @Test - public void shouldAllowUnauthenticatedEntityApplyAndListing() { - String project = "default"; - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - insecureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - List listEntitiesResponse = insecureApiClient.simpleListEntities(project); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - public void shouldAllowAuthenticatedEntityApplyAndListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - String project = "default"; - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - List listEntitiesResponse = insecureApiClient.simpleListEntities(project); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java deleted file mode 100644 index 41faee7f712..00000000000 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=http", - }) -public class CoreServiceAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/core/src/test/java/feast/core/auth/infra/JwtHelper.java b/core/src/test/java/feast/core/auth/infra/JwtHelper.java deleted file mode 100644 index 091956fb888..00000000000 --- a/core/src/test/java/feast/core/auth/infra/JwtHelper.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth.infra; - -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.JOSEObjectType; -import com.nimbusds.jose.JWSAlgorithm; -import com.nimbusds.jose.JWSHeader; -import com.nimbusds.jose.crypto.RSASSASigner; -import com.nimbusds.jose.jwk.*; -import com.nimbusds.jose.jwk.gen.RSAKeyGenerator; -import com.nimbusds.jwt.JWTClaimsSet; -import com.nimbusds.jwt.SignedJWT; -import feast.common.auth.credentials.JwtCallCredentials; -import io.grpc.*; -import java.security.interfaces.RSAPublicKey; -import java.time.Instant; -import java.util.Date; - -public final class JwtHelper { - - private static RSAKey key = null; - private JWKSet keySet; - - public JwtHelper() { - try { - key = new RSAKeyGenerator(2048).keyID("123").generate(); - } catch (JOSEException e) { - throw new RuntimeException("Could not generate RSA key"); - } - RSAKey.Builder builder = null; - try { - builder = - new RSAKey.Builder((RSAPublicKey) this.getKey().toKeyPair().getPublic()) - .keyUse(KeyUse.SIGNATURE) - .algorithm(JWSAlgorithm.RS256) - .keyID(this.getKey().getKeyID()); - } catch (JOSEException e) { - throw new RuntimeException("Could not create RSAKey builder"); - } - keySet = new JWKSet(builder.build()); - } - - public CallCredentials getCallCredentials(String email) throws JOSEException { - String jwt = createToken(email); - return new JwtCallCredentials(jwt); - } - - public String createToken(String email) throws JOSEException { - assert key != null; - JWSHeader header = - new JWSHeader.Builder(JWSAlgorithm.RS256) - .type(JOSEObjectType.JWT) - .keyID(key.getKeyID()) - .build(); - - JWTClaimsSet payload = - new JWTClaimsSet.Builder() - .issuer("me") - .audience("you") - .subject(email) - .expirationTime(Date.from(Instant.now().plusSeconds(120))) - .build(); - - SignedJWT signedJWT = new SignedJWT(header, payload); - signedJWT.sign(new RSASSASigner(key.toRSAPrivateKey())); - return signedJWT.serialize(); - } - - public RSAKey getKey() { - return key; - } - - public JWKSet getKeySet() { - return this.keySet; - } -} diff --git a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java deleted file mode 100644 index f26ce8a343f..00000000000 --- a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller; - -import static io.restassured.RestAssured.get; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.common.collect.ImmutableMap; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.restassured.RestAssured; -import io.restassured.http.ContentType; -import io.restassured.path.json.JsonPath; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.web.util.UriComponentsBuilder; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@AutoConfigureWebTestClient -public class CoreServiceRestIT extends BaseIT { - - static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleCoreClient apiClient; - @LocalServerPort private int port; - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleCoreClient(stub); - } - - @Test - public void getVersion() { - String uriString = UriComponentsBuilder.fromPath("/api/v2/version").toUriString(); - get(uriString) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("version", notNullValue()); - } - - // list projects - @Test - public void listProjects() { - // should get 2 projects - String uriString = UriComponentsBuilder.fromPath("/api/v2/projects").toUriString(); - String responseBody = - get(uriString) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List projectList = JsonPath.from(responseBody).getList("projects"); - assertEquals(projectList, List.of("default")); - } - - @Test - public void listFeatures() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/features") - .queryParam("entities", "entity1", "entity2") - .buildAndExpand() - .toString(); - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("features", aMapWithSize(2)); - - String uri2 = - UriComponentsBuilder.fromPath("/api/v2/features") - .queryParam("entities", "entity1", "entity2") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - get(uri2) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("features", aMapWithSize(2)); - } - - @Test - public void listEntities() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/entities") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List entityList = JsonPath.from(responseBody).getList("entities"); - assertEquals(entityList.size(), 2); - } - - @Test - public void listFeatureTables() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/feature-tables") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List featureTableList = JsonPath.from(responseBody).getList("tables"); - assertEquals(featureTableList.size(), 1); - } - - @BeforeEach - private void createSpecs() { - // Apply entities - EntityProto.EntitySpecV2 entitySpec1 = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - avro.shaded.com.google.common.collect.ImmutableMap.of("label_key", "label_value")); - EntityProto.EntitySpecV2 entitySpec2 = - DataGenerator.createEntitySpecV2( - "entity2", - "Entity 2 description", - ValueProto.ValueType.Enum.STRING, - avro.shaded.com.google.common.collect.ImmutableMap.of("label_key2", "label_value2")); - apiClient.simpleApplyEntity("default", entitySpec1); - apiClient.simpleApplyEntity("default", entitySpec2); - - // Apply feature table - FeatureTableProto.FeatureTableSpec featureTableSpec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - apiClient.applyFeatureTable("default", featureTableSpec); - - RestAssured.port = port; - } -} diff --git a/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/core/src/test/java/feast/core/logging/CoreLoggingIT.java deleted file mode 100644 index 0f137b46393..00000000000 --- a/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Streams; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.logging.entry.AuditLogEntryKind; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Status.Code; -import io.grpc.StatusRuntimeException; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest( - properties = { - "feast.logging.audit.enabled=true", - "feast.logging.audit.messageLogging.enabled=true", - "feast.logging.audit.messageLogging.destination=console" - }) -public class CoreLoggingIT extends BaseIT { - private static TestLogAppender testAuditLogAppender; - private static CoreServiceBlockingStub coreService; - private static CoreServiceFutureStub asyncCoreService; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int coreGrpcPort) - throws InterruptedException, ExecutionException { - LoggerContext logContext = (LoggerContext) LogManager.getContext(false); - // NOTE: As log appender state is shared across tests use a different method - // for each test and filter by method name to ensure that you only get logs - // for a specific test. - testAuditLogAppender = logContext.getConfiguration().getAppender("TestAuditLogAppender"); - - // Connect to core service. - Channel channel = - ManagedChannelBuilder.forAddress("localhost", coreGrpcPort).usePlaintext().build(); - coreService = CoreServiceGrpc.newBlockingStub(channel); - asyncCoreService = CoreServiceGrpc.newFutureStub(channel); - - // Preflight a request to core service stubs to verify connection - coreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()); - asyncCoreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()).get(); - } - - /** Check that messsage audit log are produced on service call */ - @Test - public void shouldProduceMessageAuditLogsOnCall() - throws InterruptedException, InvalidProtocolBufferException { - // Generate artifical load on feast core. - UpdateStoreRequest request = - UpdateStoreRequest.newBuilder().setStore(DataGenerator.getDefaultStore()).build(); - UpdateStoreResponse response = coreService.updateStore(request); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Check message audit logs are produced for each audit log. - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "UpdateStore"); - assertEquals(1, logJsonObjects.size()); - JsonObject logObj = logJsonObjects.get(0); - - // Extract & Check that request/response are returned correctly - String requestJson = logObj.getAsJsonObject("request").toString(); - UpdateStoreRequest.Builder gotRequest = UpdateStoreRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - UpdateStoreResponse.Builder gotResponse = UpdateStoreResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - } - - /** Check that message audit logs are produced when server encounters an error */ - @Test - public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { - // Send a bad request which should cause Core to error - ListFeatureTablesRequest request = - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) - .build(); - - boolean hasExpectedException = false; - Code statusCode = null; - try { - coreService.listFeatureTables(request); - } catch (StatusRuntimeException e) { - hasExpectedException = true; - statusCode = e.getStatus().getCode(); - } - assertTrue(hasExpectedException); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); - - assertEquals(1, logJsonObjects.size()); - JsonObject logJsonObject = logJsonObjects.get(0); - // Check correct status code is tracked on error. - assertEquals(logJsonObject.get("statusCode").getAsString(), statusCode.toString()); - } - - /** Check that expected message audit logs are produced when under load. */ - @Test - public void shouldProduceExpectedAuditLogsUnderLoad() - throws InterruptedException, ExecutionException { - // Generate artifical requests on core to simulate load. - int LOAD_SIZE = 40; // Total number of requests to send. - int BURST_SIZE = 5; // Number of requests to send at once. - - ListStoresRequest request = ListStoresRequest.getDefaultInstance(); - List responses = new LinkedList<>(); - for (int i = 0; i < LOAD_SIZE; i += 5) { - List> futures = new LinkedList<>(); - for (int j = 0; j < BURST_SIZE; j++) { - futures.add(asyncCoreService.listStores(request)); - } - - responses.addAll(Futures.allAsList(futures).get()); - } - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - - // Pull message audit logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListStores"); - assertEquals(responses.size(), logJsonObjects.size()); - - // Extract & Check that request/response are returned correctly - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - Streams.zip( - responses.stream(), - logJsonObjects.stream(), - (response, logObj) -> Pair.of(response, logObj)) - .forEach( - responseLogJsonPair -> { - ListStoresResponse response = responseLogJsonPair.getLeft(); - JsonObject logObj = responseLogJsonPair.getRight(); - - ListStoresRequest.Builder gotRequest = null; - ListStoresResponse.Builder gotResponse = null; - try { - String requestJson = logObj.getAsJsonObject("request").toString(); - gotRequest = ListStoresRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - gotResponse = ListStoresResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - }); - } - - /** - * Filter and Parse out Message Audit Logs from the given logsStrings for the given method name - */ - private List parseMessageJsonLogObjects(List logsStrings, String methodName) { - JsonParser jsonParser = new JsonParser(); - // copy to prevent concurrent modification. - return logsStrings.stream() - .map(logJSON -> jsonParser.parse(logJSON).getAsJsonObject()) - // Filter to only include message audit logs - .filter( - logObj -> - logObj - .getAsJsonPrimitive("kind") - .getAsString() - .equals(AuditLogEntryKind.MESSAGE.toString()) - // filter by method name to ensure logs from other tests do not interfere with - // test - && logObj.get("method").getAsString().equals(methodName)) - .collect(Collectors.toList()); - } -} diff --git a/core/src/test/java/feast/core/logging/TestLogAppender.java b/core/src/test/java/feast/core/logging/TestLogAppender.java deleted file mode 100644 index b1908c0db2c..00000000000 --- a/core/src/test/java/feast/core/logging/TestLogAppender.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import lombok.Getter; -import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.Core; -import org.apache.logging.log4j.core.Filter; -import org.apache.logging.log4j.core.Layout; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.appender.AbstractAppender; -import org.apache.logging.log4j.core.config.Property; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginElement; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.layout.PatternLayout; - -/** Test Log Appender used for collecting logs for testing logging. */ -@Plugin( - name = "TestLogAppender", - category = Core.CATEGORY_NAME, - elementType = Appender.ELEMENT_TYPE) -@Getter -public class TestLogAppender extends AbstractAppender { - private List logs; - - protected TestLogAppender(String name, Filter filter, Layout layout) { - super(name, filter, layout, false, new Property[] {}); - logs = new ArrayList<>(); - } - - @Override - public void append(LogEvent event) { - getLogs().add(event.getMessage().toString()); - } - - @PluginFactory - public static TestLogAppender createAppender( - @PluginAttribute("name") String name, - @PluginElement("Layout") Layout layout, - @PluginElement("Filter") final Filter filter) { - if (name == null) { - return null; - } - if (layout == null) { - layout = PatternLayout.createDefaultLayout(); - } - return new TestLogAppender(name, filter, layout); - } -} diff --git a/core/src/test/java/feast/core/metrics/CoreMetricsIT.java b/core/src/test/java/feast/core/metrics/CoreMetricsIT.java deleted file mode 100644 index f38dc75e815..00000000000 --- a/core/src/test/java/feast/core/metrics/CoreMetricsIT.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics; - -import static org.junit.Assert.assertTrue; - -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.BaseIT; -import java.io.IOException; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; - -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.security.authentication.enabled=true", - }) -public class CoreMetricsIT extends BaseIT { - private static final String METRIC_ENDPOINT = "/metrics"; - private static OkHttpClient httpClient; - @LocalServerPort private int metricsPort; - - @BeforeAll - public static void globalSetUp() { - httpClient = new OkHttpClient(); - } - - /** Test that Feast Core metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d%s", metricsPort, METRIC_ENDPOINT)) - .get() - .build(); - Response response = httpClient.newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } -} diff --git a/core/src/test/java/feast/core/model/DataSourceTest.java b/core/src/test/java/feast/core/model/DataSourceTest.java deleted file mode 100644 index 2dcbaa93cd3..00000000000 --- a/core/src/test/java/feast/core/model/DataSourceTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; - -import feast.common.it.DataGenerator; -import feast.proto.core.DataSourceProto; -import java.util.List; -import java.util.Map; -import org.junit.Test; - -public class DataSourceTest { - @Test - public void shouldSerializeFieldMappingAsJSON() { - Map expectedMap = Map.of("test", "value"); - - getTestSpecs() - .forEach( - spec -> { - DataSource source = - DataSource.fromProto(spec.toBuilder().putAllFieldMapping(expectedMap).build()); - Map actualMap = source.getFieldsMap(); - assertThat(actualMap, equalTo(actualMap)); - }); - } - - @Test - public void shouldFromProtoBeReversableWithToProto() { - getTestSpecs() - .forEach( - expectedSpec -> { - DataSourceProto.DataSource actualSpec = DataSource.fromProto(expectedSpec).toProto(); - assertThat(actualSpec, equalTo(expectedSpec)); - }); - } - - private List getTestSpecs() { - return List.of( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), - DataGenerator.createKafkaDataSourceSpec("localhost:9092", "topic", "class.path", "ts_col"), - DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), - DataGenerator.createKinesisDataSourceSpec("ap-nowhere1", "stream", "class.path", "ts_col")); - } -} diff --git a/core/src/test/java/feast/core/service/ProjectServiceTest.java b/core/src/test/java/feast/core/service/ProjectServiceTest.java deleted file mode 100644 index e09580f24e7..00000000000 --- a/core/src/test/java/feast/core/service/ProjectServiceTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import feast.core.dao.ProjectRepository; -import feast.core.model.Project; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mock; - -public class ProjectServiceTest { - - @Mock private ProjectRepository projectRepository; - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - private ProjectService projectService; - - @Before - public void setUp() { - initMocks(this); - projectRepository = mock(ProjectRepository.class); - projectService = new ProjectService(projectRepository); - } - - @Test - public void shouldCreateProjectIfItDoesntExist() { - String projectName = "project1"; - Project project = new Project(projectName); - when(projectRepository.saveAndFlush(project)).thenReturn(project); - projectService.createProject(projectName); - verify(projectRepository, times(1)).saveAndFlush(project); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldNotCreateProjectIfItExist() { - String projectName = "project1"; - when(projectRepository.existsById(projectName)).thenReturn(true); - projectService.createProject(projectName); - } - - @Test - public void shouldArchiveProjectIfItExists() { - String projectName = "project1"; - Project project = new Project(projectName); - when(projectRepository.findById(projectName)).thenReturn(Optional.of(project)); - projectService.archiveProject(projectName); - verify(projectRepository, times(1)).saveAndFlush(project); - } - - @Test - public void shouldNotArchiveDefaultProject() { - expectedException.expect(IllegalArgumentException.class); - this.projectService.archiveProject(Project.DEFAULT_NAME); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldNotArchiveProjectIfItIsAlreadyArchived() { - String projectName = "project1"; - when(projectRepository.findById(projectName)).thenReturn(Optional.empty()); - projectService.archiveProject(projectName); - } - - @Test - public void shouldListProjects() { - String projectName = "project1"; - Project project = new Project(projectName); - List expected = Arrays.asList(project); - when(projectRepository.findAllByArchivedIsFalse()).thenReturn(expected); - List actual = projectService.listProjects(); - Assert.assertEquals(expected, actual); - } -} diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java deleted file mode 100644 index 8851d875ae8..00000000000 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ /dev/null @@ -1,903 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static com.jayway.jsonassert.impl.matcher.IsMapContainingKey.hasKey; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.hamcrest.core.IsIterableContaining.hasItem; -import static org.junit.Assert.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.google.protobuf.Duration; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.common.util.TestUtil; -import feast.proto.core.*; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.util.*; -import java.util.stream.IntStream; -import org.apache.commons.lang3.tuple.Triple; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; - -@SpringBootTest -public class SpecServiceIT extends BaseIT { - - static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleCoreClient apiClient; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleCoreClient(stub); - } - - private FeatureTableProto.FeatureTableSpec example1; - private FeatureTableProto.FeatureTableSpec example2; - - @BeforeEach - public void initState() { - - EntityProto.EntitySpecV2 entitySpec1 = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - EntityProto.EntitySpecV2 entitySpec2 = - DataGenerator.createEntitySpecV2( - "entity2", - "Entity 2 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key2", "label_value2")); - apiClient.simpleApplyEntity("default", entitySpec1); - apiClient.simpleApplyEntity("default", entitySpec2); - - example1 = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - example2 = - DataGenerator.createFeatureTableSpec( - "featuretable2", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature3", ValueProto.ValueType.Enum.STRING); - put("feature4", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key4", "feat_value4")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - apiClient.applyFeatureTable("default", example1); - apiClient.applyFeatureTable("default", example2); - apiClient.simpleApplyEntity( - "project1", - DataGenerator.createEntitySpecV2( - "entity3", - "Entity 3 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key2", "label_value2"))); - apiClient.updateStore(DataGenerator.getDefaultStore()); - } - - @Nested - class ListEntities { - @Test - public void shouldFilterEntitiesByLabels() { - List entities = - apiClient.simpleListEntities("", ImmutableMap.of("label_key2", "label_value2")); - - assertThat(entities, hasSize(1)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity2"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - List entities = apiClient.simpleListEntities(""); - - assertThat(entities, hasSize(2)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity1"))))); - } - - @Test - public void shouldFilterEntitiesByProjectAndLabels() { - List entities = - apiClient.simpleListEntities("project1", ImmutableMap.of("label_key2", "label_value2")); - - assertThat(entities, hasSize(1)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity3"))))); - } - - @Test - public void shouldThrowExceptionGivenWildcardProject() { - CoreServiceProto.ListEntitiesRequest.Filter filter = - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder().setProject("default*").build(); - StatusRuntimeException exc = - assertThrows(StatusRuntimeException.class, () -> apiClient.simpleListEntities(filter)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", - filter.getProject()))); - } - } - - @Nested - class ListFeatureTables { - @Test - public void shouldFilterFeatureTablesByProjectAndLabels() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .putAllLabels(ImmutableMap.of("feat_key2", "feat_value2")) - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - assertThat(featureTables, hasSize(1)); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable1"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - assertThat(featureTables, hasSize(2)); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable1"))))); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable2"))))); - } - - @Test - public void shouldThrowExceptionGivenWildcardProject() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default*") - .build(); - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleListFeatureTables(filter)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters and underscores.", - filter.getProject()))); - } - } - - @Nested - class ApplyEntity { - @Test - public void shouldThrowExceptionGivenEntityWithDash() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyEntity( - "default", - DataGenerator.createEntitySpecV2( - "dash-entity", - "Dash Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("test_key", "test_value")))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: invalid value for %s resource, %s: %s", - "entity", - "dash-entity", - "argument must only contain alphanumeric characters and underscores."))); - } - - @Test - public void shouldThrowExceptionIfTypeChanged() { - String projectName = "default"; - - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity description", - ValueProto.ValueType.Enum.FLOAT, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleApplyEntity("default", spec)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: You are attempting to change the type of this entity in %s project from %s to %s. This isn't allowed. Please create a new entity.", - "default", "STRING", spec.getValueType()))); - } - - @Test - public void shouldReturnEntityIfEntityHasNotChanged() { - String projectName = "default"; - EntityProto.EntitySpecV2 spec = apiClient.simpleGetEntity(projectName, "entity1").getSpec(); - - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity(projectName, spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldApplyEntityIfNotExists() { - String projectName = "default"; - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "new_entity", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity(projectName, spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldCreateProjectWhenNotAlreadyExists() { - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "new_entity2", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("key1", "val1")); - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity("new_project", spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldFailWhenProjectIsArchived() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyEntity( - "archived", - DataGenerator.createEntitySpecV2( - "new_entity3", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("key1", "val1")))); - assertThat(exc.getMessage(), equalTo("INTERNAL: Project is archived: archived")); - } - - @Test - public void shouldUpdateLabels() { - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value", "label_key2", "label_value2")); - - CoreServiceProto.ApplyEntityResponse response = apiClient.simpleApplyEntity("default", spec); - - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - } - } - - @Nested - class UpdateStore { - @Test - public void shouldUpdateStoreIfConfigChanges() { - StoreProto.Store defaultStore = DataGenerator.getDefaultStore(); - - StoreProto.Store updatedStore = - DataGenerator.createStore( - defaultStore.getName(), - defaultStore.getType(), - ImmutableList.of(Triple.of("project1", "*", false))); - - CoreServiceProto.UpdateStoreResponse response = apiClient.updateStore(updatedStore); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.UpdateStoreResponse.Status.UPDATED)); - } - - @Test - public void shouldDoNothingIfNoChange() { - CoreServiceProto.UpdateStoreResponse response = - apiClient.updateStore(DataGenerator.getDefaultStore()); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.UpdateStoreResponse.Status.NO_CHANGE)); - } - } - - @Nested - class GetEntity { - @Test - public void shouldThrowExceptionGivenMissingEntity() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleGetEntity("default", "")); - - assertThat(exc.getMessage(), equalTo("INVALID_ARGUMENT: No entity name provided")); - } - - public void shouldRetrieveFromDefaultIfProjectNotSpecified() { - String entityName = "entity1"; - EntityProto.Entity entity = apiClient.simpleGetEntity("", entityName); - - assertThat(entity.getSpec().getName(), equalTo(entityName)); - } - } - - @Nested - class GetFeatureTable { - @Test - public void shouldThrowExceptionGivenNoSuchFeatureTable() { - String projectName = "default"; - String featureTableName = "invalid_table"; - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.simpleGetFeatureTable(projectName, featureTableName)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: No such Feature Table: (project: %s, name: %s)", - projectName, featureTableName))); - } - - @Test - public void shouldReturnFeatureTableIfExists() { - FeatureTableProto.FeatureTable featureTable = - apiClient.simpleGetFeatureTable("default", "featuretable1"); - - assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), example1)); - } - } - - @Nested - class ListStores { - @Test - public void shouldReturnAllStoresIfNoNameProvided() { - apiClient.updateStore(DataGenerator.getDefaultStore()); - apiClient.updateStore( - DataGenerator.createStore( - "data", StoreProto.Store.StoreType.REDIS, Collections.emptyList())); - - List actual = - stub.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter(CoreServiceProto.ListStoresRequest.Filter.newBuilder().build()) - .build()) - .getStoreList(); - - assertThat(actual, hasSize(2)); - assertThat(actual, hasItem(hasProperty("name", equalTo("test-store")))); - assertThat(actual, hasItem(hasProperty("name", equalTo("data")))); - } - - @Test - public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - stub.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter( - CoreServiceProto.ListStoresRequest.Filter.newBuilder() - .setName("unknown") - .build()) - .build())); - - assertThat(exc.getMessage(), equalTo("INTERNAL: Store with name 'unknown' not found")); - } - } - - @Nested - class ListFeatures { - @Test - public void shouldFilterFeaturesByEntitiesAndLabels() { - // Case 1: Only filter by entities - Map result1 = - apiClient.simpleListFeatures("default", "entity1", "entity2"); - - assertThat(result1, aMapWithSize(4)); - assertThat(result1, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result1, hasKey(equalTo("featuretable1:feature2"))); - assertThat(result1, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result1, hasKey(equalTo("featuretable2:feature4"))); - - // Case 2: Filter by entities and labels - Map result2 = - apiClient.simpleListFeatures( - "default", - ImmutableMap.of("feat_key2", "feat_value2"), - ImmutableList.of("entity1", "entity2")); - - assertThat(result2, aMapWithSize(2)); - assertThat(result2, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result2, hasKey(equalTo("featuretable1:feature2"))); - - // Case 3: Filter by labels - Map result3 = - apiClient.simpleListFeatures( - "default", ImmutableMap.of("feat_key4", "feat_value4"), Collections.emptyList()); - - assertThat(result3, aMapWithSize(2)); - assertThat(result3, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result3, hasKey(equalTo("featuretable2:feature4"))); - - // Case 4: Filter by nothing, except project - Map result4 = - apiClient.simpleListFeatures("project1", ImmutableMap.of(), Collections.emptyList()); - - assertThat(result4, aMapWithSize(0)); - - // Case 5: Filter by nothing; will use default project - Map result5 = - apiClient.simpleListFeatures("", ImmutableMap.of(), Collections.emptyList()); - - assertThat(result5, aMapWithSize(4)); - assertThat(result5, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result5, hasKey(equalTo("featuretable1:feature2"))); - assertThat(result5, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result5, hasKey(equalTo("featuretable2:feature4"))); - - // Case 6: Filter by mismatched entity - Map result6 = - apiClient.simpleListFeatures("default", ImmutableMap.of(), ImmutableList.of("entity1")); - assertThat(result6, aMapWithSize(0)); - } - } - - @Nested - public class ApplyFeatureTable { - private FeatureTableSpec getTestSpec() { - return example1 - .toBuilder() - .setName("apply_test") - .setStreamSource( - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col")) - .build(); - } - - @Test - public void shouldApplyNewValidTable() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - - assertTrue(TestUtil.compareFeatureTableSpec(table.getSpec(), getTestSpec())); - assertThat(table.getMeta().getRevision(), equalTo(0L)); - } - - @Test - public void shouldUpdateExistingTableWithValidSpec() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - - FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .clearFeatures() - .addFeatures( - DataGenerator.createFeatureSpecV2( - "feature5", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) - .setStreamSource( - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "new_topic", "new.class", "ts_col")) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - assertThat(updatedTable.getMeta().getRevision(), equalTo(table.getMeta().getRevision() + 1L)); - } - - @Test - public void shouldUpdateFeatureTableOnEntityChange() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec().toBuilder().clearEntities().addEntities("entity1").build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnMaxAgeChange() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .setMaxAge(Duration.newBuilder().setSeconds(600).build()) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnFeatureTypeChange() { - int featureIdx = - IntStream.range(0, getTestSpec().getFeaturesCount()) - .filter(i -> getTestSpec().getFeatures(i).getName().equals("feature2")) - .findFirst() - .orElse(-1); - - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .setFeatures( - featureIdx, - DataGenerator.createFeatureSpecV2( - "feature2", ValueProto.ValueType.Enum.STRING_LIST, ImmutableMap.of())) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnFeatureAddition() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .addFeatures( - DataGenerator.createFeatureSpecV2( - "feature6", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldNotUpdateIfNoChanges() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", getTestSpec()); - - assertThat(updatedTable.getMeta().getRevision(), equalTo(table.getMeta().getRevision())); - } - - @Test - public void shouldErrorOnMissingBatchSource() { - FeatureTableProto.FeatureTableSpec spec = - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("event_timestamp", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .clearBatchSource() - .build(); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.applyFeatureTable("default", spec)); - - assertThat( - exc.getMessage(), - equalTo("INVALID_ARGUMENT: FeatureTable batch source cannot be empty.")); - } - - @Test - public void shouldErrorOnInvalidBigQueryTableRef() { - String invalidTableRef = "invalid.bq:path"; - FeatureTableProto.FeatureTableSpec spec = - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("feature", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createBigQueryDataSourceSpec(invalidTableRef, "ts_col", "")) - .build(); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.applyFeatureTable("default", spec)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for FeatureTable resource, %s: argument must be in the form of .", - invalidTableRef))); - } - - @Test - public void shouldErrorOnReservedNames() { - // Reserved name used as feature name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("event_timestamp", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - - // Reserved name used in as entity name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("created_timestamp"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnInvalidName() { - // Invalid feature table name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "f-t", - List.of("entity1"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - - // Invalid feature name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("feature-1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnNotFoundEntityName() { - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft1", - List.of("entity_not_found"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnArchivedProject() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "archived", - DataGenerator.createFeatureTableSpec( - "ft1", - List.of("entity1", "entity2"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - } - - @Nested - public class DeleteFeatureTable { - - @Test - public void shouldReturnNoTables() { - String projectName = "default"; - String featureTableName = "featuretable1"; - - apiClient.deleteFeatureTable(projectName, featureTableName); - - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .putLabels("feat_key2", "feat_value2") - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.simpleGetFeatureTable(projectName, featureTableName)); - - assertThat(featureTables.size(), equalTo(0)); - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: Feature Table has been deleted: (project: %s, name: %s)", - projectName, featureTableName))); - } - - @Test - public void shouldUpdateDeletedTable() { - String projectName = "default"; - String featureTableName = "featuretable1"; - - apiClient.deleteFeatureTable(projectName, featureTableName); - - FeatureTableSpec featureTableSpec = - DataGenerator.createFeatureTableSpec( - featureTableName, - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature3", ValueProto.ValueType.Enum.INT64); - } - }, - 7200, - ImmutableMap.of("feat_key3", "feat_value3")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - apiClient.applyFeatureTable(projectName, featureTableSpec); - - FeatureTableProto.FeatureTable featureTable = - apiClient.simpleGetFeatureTable(projectName, featureTableName); - - assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), featureTableSpec)); - } - - @Test - public void shouldErrorIfTableNotExist() { - String projectName = "default"; - String featureTableName = "nonexistent_table"; - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.deleteFeatureTable(projectName, featureTableName)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: No such Feature Table: (project: %s, name: %s)", - projectName, featureTableName))); - } - } -} diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java deleted file mode 100644 index c44bf50129e..00000000000 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.*; - -import com.google.protobuf.Timestamp; -import java.util.*; -import org.junit.Test; - -public class TypeConversionTest { - @Test - public void convertTimeStampShouldCorrectlyConvertDateToProtobufTimestamp() { - Date date = new Date(1000); - Timestamp expected = Timestamp.newBuilder().setSeconds(1).build(); - assertThat(TypeConversion.convertTimestamp(date), equalTo(expected)); - } - - @Test - public void convertTagStringToListShouldConvertTagStringToList() { - String input = "value1,value2"; - List expected = Arrays.asList("value1", "value2"); - assertThat(TypeConversion.convertTagStringToList(input), equalTo(expected)); - } - - @Test - public void convertTagStringToListShouldReturnEmptyListForEmptyString() { - String input = ""; - List expected = Collections.emptyList(); - assertThat(TypeConversion.convertTagStringToList(input), equalTo(expected)); - } - - @Test - public void convertJsonStringToMapShouldConvertJsonStringToMap() { - String input = "{\"key\": \"value\"}"; - Map expected = new HashMap<>(); - expected.put("key", "value"); - assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); - } - - @Test - public void convertJsonStringToMapShouldReturnEmptyMapForEmptyJson() { - String input = "{}"; - Map expected = Collections.emptyMap(); - assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); - } - - @Test - public void convertMapToJsonStringShouldReturnJsonStringForGivenMap() { - Map input = new HashMap<>(); - input.put("key", "value"); - assertThat( - TypeConversion.convertMapToJsonString(input), hasJsonPath("$.key", equalTo("value"))); - } - - @Test - public void convertMapToJsonStringShouldReturnEmptyJsonForAnEmptyMap() { - Map input = new HashMap<>(); - assertThat(TypeConversion.convertMapToJsonString(input), equalTo("{}")); - } -} diff --git a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java deleted file mode 100644 index 842e4c657d6..00000000000 --- a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; - -import feast.common.it.DataGenerator; -import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; -import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import java.util.Map; -import org.junit.Test; - -public class DataSourceValidatorTest { - - @Test(expected = UnsupportedOperationException.class) - public void shouldErrorIfSourceTypeUnsupported() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap().get(BATCH_FILE).toBuilder().setType(SourceType.INVALID).build(); - DataSourceValidator.validate(badSpec); - } - - @Test - public void shouldPassValidSpecs() { - getTestSpecsMap().values().forEach(DataSourceValidator::validate); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfBadBigQueryTableRef() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap() - .get(BATCH_BIGQUERY) - .toBuilder() - .setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef("bad:/ref").build()) - .build(); - DataSourceValidator.validate(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfBadClassPath() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap() - .get(STREAM_KAFKA) - .toBuilder() - .setKafkaOptions( - KafkaOptions.newBuilder() - .setMessageFormat(DataGenerator.createProtoFormat(".bad^path")) - .build()) - .build(); - DataSourceValidator.validate(badSpec); - - badSpec = - getTestSpecsMap() - .get(STREAM_KINESIS) - .toBuilder() - .setKinesisOptions( - KinesisOptions.newBuilder() - .setRecordFormat(DataGenerator.createProtoFormat(".bad^path")) - .build()) - .build(); - DataSourceValidator.validate(badSpec); - } - - private Map getTestSpecsMap() { - return Map.of( - BATCH_FILE, DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), - BATCH_BIGQUERY, - DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), - STREAM_KINESIS, - DataGenerator.createKinesisDataSourceSpec( - "ap-nowhere1", "stream", "class.path", "ts_col"), - STREAM_KAFKA, - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col")); - } -} diff --git a/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java deleted file mode 100644 index 8410799dc68..00000000000 --- a/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.proto.types.ValueProto.ValueType.Enum.*; - -import feast.common.it.DataGenerator; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.junit.Test; - -public class FeatureTableValidatorTest { - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfLabelsHasEmptyKey() { - Map badLabels = Map.of("", "empty"); - FeatureTableSpec badSpec = getTestSpec().toBuilder().putAllLabels(badLabels).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfFeaturesLabelsHasEmptyKey() { - Map badLabels = Map.of("", "empty"); - - List badFeatureSpecs = - getTestSpec().getFeaturesList().stream() - .map(featureSpec -> featureSpec.toBuilder().putAllLabels(badLabels).build()) - .collect(Collectors.toList()); - FeatureTableSpec badSpec = getTestSpec().toBuilder().addAllFeatures(badFeatureSpecs).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfUsedReservedName() { - FeatureTableSpec badSpec = - getTestSpec().toBuilder().addAllEntities(FeatureTableValidator.RESERVED_NAMES).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfNamesUsedNotUnique() { - FeatureTableSpec badSpec = - DataGenerator.createFeatureTableSpec( - "driver", List.of("region"), Map.of("region", STRING), 3600, Map.of()); - FeatureTableValidator.validateSpec(badSpec); - } - - private FeatureTableSpec getTestSpec() { - return DataGenerator.createFeatureTableSpec( - "driver", List.of("driver_id"), Map.of("n_drivers", INT64), 3600, Map.of()); - } -} diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java deleted file mode 100644 index 17332123d21..00000000000 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.checkLowerSnakeCase; -import static feast.core.validators.Matchers.checkUpperSnakeCase; -import static feast.core.validators.Matchers.checkValidClassPath; - -import com.google.common.base.Strings; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -public class MatchersTest { - @Rule public final ExpectedException exception = ExpectedException.none(); - - @Test - public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { - String in = "REDIS_DB"; - checkUpperSnakeCase(in, "featuretable"); - } - - @Test - public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { - String in = "REDIS1"; - checkUpperSnakeCase(in, "featuretable"); - } - - @Test - public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( - Strings.lenientFormat( - "invalid value for %s resource, %s: %s", - "featuretable", - "redis", - "argument must be in upper snake case, and cannot include any special characters.")); - String in = "redis"; - checkUpperSnakeCase(in, "featuretable"); - } - - @Test - public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() { - String in = "feature_name_v1"; - checkLowerSnakeCase(in, "feature"); - } - - @Test - public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage( - Strings.lenientFormat( - "invalid value for %s resource, %s: %s", - "feature", - "Invalid_feature name", - "argument must be in lower snake case, and cannot include any special characters.")); - String in = "Invalid_feature name"; - checkLowerSnakeCase(in, "feature"); - } - - @Test - public void checkValidClassPathSuccess() { - checkValidClassPath("com.example.foo", "FeatureTable"); - checkValidClassPath("com.example", "FeatureTable"); - } - - @Test - public void checkValidClassPathEmpty() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("", "FeatureTable"); - } - - @Test - public void checkValidClassPathDigits() { - exception.expect(IllegalArgumentException.class); - checkValidClassPath("123", "FeatureTable"); - } -} diff --git a/core/src/test/resources/application-it.properties b/core/src/test/resources/application-it.properties deleted file mode 100644 index 7e71d8293fb..00000000000 --- a/core/src/test/resources/application-it.properties +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -grpc.server.port=6666 - -feast.security.authentication.enabled = false -feast.security.authorization.enabled = false - -feast.jobs.enabled=false - -spring.datasource.hikari.maximum-pool-size=40 -spring.main.allow-bean-definition-overriding=true - diff --git a/core/src/test/resources/application.properties b/core/src/test/resources/application.properties deleted file mode 100644 index cbc2e8ef0d1..00000000000 --- a/core/src/test/resources/application.properties +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -grpc.port=${GRPC_PORT:6565} - -feast.security.authentication.enabled = true -feast.security.disableRestControllerAuth = true -feast.security.authorization.enabled = false - -feast.core.projectId = ${PROJECT_ID:} -feast.core.datasetPrefix = ${DATASET_PREFIX:fs} - -feast.jobs.workspace=${JOB_WORKSPACE} -feast.jobs.runner=${JOB_RUNNER:DirectRunner} -feast.jobs.options=${JOB_OPTIONS:{}} -feast.jobs.executable=${JOB_EXECUTABLE:feast-ingestion.jar} - -feast.jobs.dataflow.projectId = ${DATAFLOW_PROJECT_ID:} -feast.jobs.dataflow.location = ${DATAFLOW_LOCATION:} - -feast.jobs.flink.configDir = ${FLINK_CONF_DIR:/etc/flink/flink-1.5.5/conf} -feast.jobs.flink.masterUrl = ${FLINK_MASTER_URL:localhost:8081} - -feast.jobs.monitor.period = ${JOB_MONITOR_PERIOD_MS:5000} -feast.jobs.monitor.initialDelay = ${JOB_MONITOR_INITIAL_DELAY_MS:60000} - -feast.store.serving.type = ${STORE_SERVING_TYPE:} -feast.store.serving.options = ${STORE_SERVING_OPTIONS:{}} -feast.store.warehouse.type = ${STORE_WAREHOUSE_TYPE:} -feast.store.warehouse.options = ${STORE_WAREHOUSE_OPTIONS:{}} -feast.store.errors.type = ${STORE_ERRORS_TYPE:} -feast.store.errors.options = ${STORE_ERRORS_OPTIONS:{}} - -statsd.host= ${STATSD_HOST:localhost} -statsd.port= ${STATSD_PORT:8125} - -management.metrics.export.simple.enabled=false -management.metrics.export.statsd.enabled=true -management.metrics.export.statsd.host=${STATSD_HOST:localhost} -management.metrics.export.statsd.port=${STATSD_PORT:8125} diff --git a/core/src/test/resources/keto/docker-compose.yml b/core/src/test/resources/keto/docker-compose.yml deleted file mode 100644 index 4d0fc43021b..00000000000 --- a/core/src/test/resources/keto/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3' -services: - keto: - depends_on: - - db - - migrations - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@db:5432/keto?sslmode=disable - command: - - serve - ports: - - 4466 - - db: - image: bitnami/postgresql:9.6 - environment: - - POSTGRESQL_USERNAME=keto - - POSTGRESQL_PASSWORD=keto - - POSTGRESQL_DATABASE=keto - - migrations: - depends_on: - - db - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@db:5432/keto?sslmode=disable - command: - - migrate - - sql - - -e - - adaptor: - depends_on: - - keto - image: gcr.io/kf-feast/feast-keto-auth-server:latest - environment: - SERVER_PORT: 8080 - KETO_URL: http://keto:4466 - ports: - - 8080 - restart: on-failure \ No newline at end of file diff --git a/core/src/test/resources/log4j2.xml b/core/src/test/resources/log4j2.xml deleted file mode 100644 index 9f785952e0c..00000000000 --- a/core/src/test/resources/log4j2.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8e..00000000000 --- a/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file diff --git a/core/src/test/resources/sql/expQuery1.sql b/core/src/test/resources/sql/expQuery1.sql deleted file mode 100644 index 019f5d9cbd4..00000000000 --- a/core/src/test/resources/sql/expQuery1.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT - id, - event_timestamp, - feature1, - feature2, - feature3 -FROM - `project.dataset.myentity` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 100 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQuery2.sql b/core/src/test/resources/sql/expQuery2.sql deleted file mode 100644 index e9b212d20ac..00000000000 --- a/core/src/test/resources/sql/expQuery2.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - id, - event_timestamp, - feature1 -FROM - `project.dataset.myentity` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 1000 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQueryWithJobIdFilter.sql b/core/src/test/resources/sql/expQueryWithJobIdFilter.sql deleted file mode 100644 index 96c076b6a6d..00000000000 --- a/core/src/test/resources/sql/expQueryWithJobIdFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 AND feature2 = "HELLO" AND job_id = "1234567890" -LIMIT 1000 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql b/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql deleted file mode 100644 index 8769fe44af7..00000000000 --- a/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 AND feature2 = "HELLO" -LIMIT 1000 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQueryWithNumberFilter.sql b/core/src/test/resources/sql/expQueryWithNumberFilter.sql deleted file mode 100644 index 6b199b7c4b8..00000000000 --- a/core/src/test/resources/sql/expQueryWithNumberFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 -LIMIT 1000 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQueryWithStringFilter.sql b/core/src/test/resources/sql/expQueryWithStringFilter.sql deleted file mode 100644 index 8c0a3805041..00000000000 --- a/core/src/test/resources/sql/expQueryWithStringFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = "10" -LIMIT 1000 \ No newline at end of file diff --git a/datatypes/java/README.md b/datatypes/java/README.md deleted file mode 100644 index 30b568e92a8..00000000000 --- a/datatypes/java/README.md +++ /dev/null @@ -1,55 +0,0 @@ -Feast Data Types for Java -========================= - -This module produces Java class files for Feast's data type and gRPC service -definitions, from Protobuf IDL. These are used across Feast components for wire -interchange, contracts, etc. - -End users of Feast will be best served by our Java SDK which adds higher-level -conveniences, but the data types are published independently for custom needs, -without any additional dependencies the SDK may add. - -Dependency Coordinates ----------------------- - -```xml - - dev.feast - datatypes-java - 0.10.0-SNAPSHOT - -``` - -Use the version corresponding to the Feast release you have deployed in your -environment—see the [Feast release notes] for details. - -[Feast release notes]: ../../CHANGELOG.md - -Using the `.proto` Definitions ------------------------------- - -The `.proto` definitions are packaged as resources within the Maven artifact, -which may be useful to `include` them in dependent Protobuf definitions in a -downstream project, or for other JVM languages to consume from their builds to -generate more idiomatic bindings. - -Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle] -either for `include` or to compile with a different `protoc` plugin than Java. - -[sbt-protoc] offers similar functionality for sbt/Scala. - -[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies -[sbt-protoc]: https://github.com/thesamet/sbt-protoc - -Releases --------- - -The module is published to Maven Central upon each release of Feast (since -v0.3.7). - -For developers, the publishing process is automated along with the Java SDK by -[the `publish-java-sdk` build task in Prow][prow task], where you can see how -it works. Artifacts are staged to Sonatype where a maintainer needs to take a -release action for them to go live on Maven Central. - -[prow task]: https://github.com/feast-dev/feast/blob/17e7dca8238aae4dcbf0ff9f0db5d80ef8e035cf/.prow/config.yaml#L166-L192 diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml deleted file mode 100644 index dd2a162c01c..00000000000 --- a/datatypes/java/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - 4.0.0 - - Feast Data Types for Java - - Data types and service contracts used throughout Feast components and - their interchanges. These are generated from Protocol Buffers and gRPC - definitions included in the package. - - datatypes-java - - - dev.feast - feast-parent - ${revision} - ../.. - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - - javax.annotation - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - true - - com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} - - grpc-java - - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - - - - - - compile - compile-custom - test-compile - - - - - - - - - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - - io.grpc - grpc-core - - - io.grpc - grpc-protobuf - - - io.grpc - grpc-services - - - io.grpc - grpc-stub - - - - javax.annotation - javax.annotation-api - - - - diff --git a/datatypes/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast deleted file mode 120000 index 463e4045de1..00000000000 --- a/datatypes/java/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../../protos/feast \ No newline at end of file diff --git a/datatypes/java/src/main/proto/tensorflow_metadata b/datatypes/java/src/main/proto/tensorflow_metadata deleted file mode 120000 index a633bb850f3..00000000000 --- a/datatypes/java/src/main/proto/tensorflow_metadata +++ /dev/null @@ -1 +0,0 @@ -../../../../../protos/tensorflow_metadata \ No newline at end of file diff --git a/docs/coverage/java/pom.xml b/docs/coverage/java/pom.xml deleted file mode 100644 index 21a75debd2c..00000000000 --- a/docs/coverage/java/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - - - - dev.feast - feast-parent - ${revision} - ../../.. - - - Feast Coverage Java - feast-coverage - - - true - - - - - dev.feast - feast-storage-api - ${project.version} - - - - dev.feast - feast-storage-connector-redis - ${project.version} - - - - dev.feast - feast-core - ${project.version} - - - - dev.feast - feast-serving - ${project.version} - - - - dev.feast - feast-sdk - ${project.version} - - - - - - - org.jacoco - jacoco-maven-plugin - - - report-aggregate - prepare-package - - report-aggregate - - - - - - - - diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile deleted file mode 100644 index c462673780b..00000000000 --- a/infra/docker/core/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# ============================================================ -# Build stage 1: Builder -# ============================================================ - -FROM maven:3.6-jdk-11 as builder - -WORKDIR /build - -COPY pom.xml . -COPY datatypes/java/pom.xml datatypes/java/pom.xml -COPY common/pom.xml common/pom.xml -COPY core/pom.xml core/pom.xml -COPY serving/pom.xml serving/pom.xml -COPY storage/api/pom.xml storage/api/pom.xml -COPY storage/connectors/pom.xml storage/connectors/pom.xml -COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY sdk/java/pom.xml sdk/java/pom.xml -COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml -COPY protos/ protos/ - -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" -COPY pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - -COPY . . - -ARG VERSION=dev -RUN mvn --also-make --projects core -Drevision=$VERSION \ - -DskipUTs=true --batch-mode clean package - -# -# Download grpc_health_probe to run health check for Feast Serving -# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ -# -RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ - -O /usr/bin/grpc-health-probe && \ - chmod +x /usr/bin/grpc-health-probe - -# ============================================================ -# Build stage 2: Production -# ============================================================ - -FROM openjdk:11-jre as production -ARG VERSION=dev - -COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar -COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe - -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-core.jar"] diff --git a/infra/docker/core/Dockerfile.debug b/infra/docker/core/Dockerfile.debug deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/infra/docker/core/Dockerfile.dev b/infra/docker/core/Dockerfile.dev deleted file mode 100644 index 70de4d9339d..00000000000 --- a/infra/docker/core/Dockerfile.dev +++ /dev/null @@ -1,8 +0,0 @@ -FROM openjdk:11-jre -ARG REVISION=dev -ADD $PWD/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-core.jar"] diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile deleted file mode 100644 index 395b344c7c1..00000000000 --- a/infra/docker/serving/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# ============================================================ -# Build stage 1: Builder -# ============================================================ - -FROM maven:3.6-jdk-11 as builder - -WORKDIR /build - -COPY pom.xml . -COPY datatypes/java/pom.xml datatypes/java/pom.xml -COPY common/pom.xml common/pom.xml -COPY core/pom.xml core/pom.xml -COPY serving/pom.xml serving/pom.xml -COPY storage/api/pom.xml storage/api/pom.xml -COPY storage/connectors/pom.xml storage/connectors/pom.xml -COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY sdk/java/pom.xml sdk/java/pom.xml -COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml -COPY protos/ protos/ - -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" -COPY pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - -COPY . . - -ARG VERSION=dev -RUN mvn --also-make --projects serving -Drevision=$VERSION \ - -DskipUTs=true --batch-mode clean package -# -# Download grpc_health_probe to run health check for Feast Serving -# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ -# -RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ - -O /usr/bin/grpc-health-probe && \ - chmod +x /usr/bin/grpc-health-probe - -# ============================================================ -# Build stage 2: Production -# ============================================================ - -FROM amazoncorretto:11 as production -ARG VERSION=dev -COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar -COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe -CMD ["java",\ - "-Xms1g",\ - "-Xmx4g",\ - "-jar",\ - "/opt/feast/feast-serving.jar"] diff --git a/infra/docker/serving/Dockerfile.dev b/infra/docker/serving/Dockerfile.dev deleted file mode 100644 index 93bbbbb7185..00000000000 --- a/infra/docker/serving/Dockerfile.dev +++ /dev/null @@ -1,15 +0,0 @@ -FROM openjdk:11-jre as production -ARG REVISION=dev -# -# Download grpc_health_probe to run health check for Feast Serving -# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ -# -RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ - -O /usr/bin/grpc-health-probe && \ - chmod +x /usr/bin/grpc-health-probe -ADD $PWD/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar -CMD ["java",\ - "-Xms1024m",\ - "-Xmx1024m",\ - "-jar",\ - "/opt/feast/feast-serving.jar"] diff --git a/infra/scripts/validate-version-consistency.sh b/infra/scripts/validate-version-consistency.sh deleted file mode 100755 index 30e294ee416..00000000000 --- a/infra/scripts/validate-version-consistency.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# This script will scan through a list of files to validate that all versions are consistent with -# - Master version: version set in maven (could be snapshot) -# - Release version: 'dev' on master, Lastest tag on release branches. -# - Stable Version: Highest stable tag. release candidates not included. -# Usage: ./validate-version-consistency.sh -# Optionaly set TARGET_MERGE_BRANCH var to the target merge branch to lint -# versions against the given merge branch. -set -e - -source infra/scripts/setup-common-functions.sh - -# Fetch tags and current branch -git fetch --prune --unshallow --tags || true -BRANCH_NAME=${TARGET_MERGE_BRANCH-$(git rev-parse --abbrev-ref HEAD)} - -# Matches (ie vMAJOR.MINOR-branch) release branch names -RELEASE_BRANCH_REGEX="^v[0-9]+\.[0-9]+-branch$" - -# Determine the current Feast version from Maven (pom.xml) -export FEAST_MASTER_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -[[ -z "$FEAST_MASTER_VERSION" ]] && { - echo "$FEAST_MASTER_VERSION is missing, please check pom.xml and maven" - exit 1 -} -echo "Linting Master Version: $FEAST_MASTER_VERSION" - -# Determine Last release tag relative to current branch -if [ $BRANCH_NAME = "master" ] -then - # Use development version - FEAST_RELEASE_VERSION="develop" -elif echo "$BRANCH_NAME" | grep -P $RELEASE_BRANCH_REGEX &>/dev/null -then - # Use last release tag tagged on the release branch - LAST_MERGED_TAG=$(get_tag_release -m) - FEAST_RELEASE_VERSION=${LAST_MERGED_TAG#"v"} -else - # Do not enforce version linting as we don't know if the target merge branch FEAST_RELEASE_VERSION="_ANY" - FEAST_RELEASE_VERSION="_ANY" - echo "WARNING: Skipping docker version lint" -fi -[[ -z "$FEAST_RELEASE_VERSION" ]] && { - echo "FEAST_RELEASE_VERSION is missing" - exit 1 -} -export FEAST_RELEASE_VERSION -echo "Linting Release Version: $FEAST_RELEASE_VERSION" - -# Determine highest stable version (no release candidates) relative to current branch. -# Regular expression for matching stable tags in the format vMAJOR.MINOR.PATCH -STABLE_TAG_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" -if [ $BRANCH_NAME = "master" ] -then - # Use last stable tag repo wide - LAST_STABLE_TAG=$(get_tag_release -s) - FEAST_STABLE_VERSION=${LAST_STABLE_TAG#"v"} -elif echo "$BRANCH_NAME" | grep -P $RELEASE_BRANCH_REGEX &>/dev/null -then - # Use last stable tag tagged on the release branch - LAST_STABLE_MERGE_TAG=$(get_tag_release -sm) - FEAST_STABLE_VERSION=${LAST_STABLE_MERGE_TAG#"v"} -else - # Do not enforce version linting as we don't know if the target merge branch - FEAST_STABLE_VERSION="_ANY" - echo "WARNING: Skipping stable version lint" -fi -[[ -z "$FEAST_STABLE_VERSION" ]] && { - echo "FEAST_STABLE_VERSION is missing" - exit 1 -} -export FEAST_STABLE_VERSION -echo "Linting Stable Version: $FEAST_STABLE_VERSION" - -# List of files to validate with master version (from pom.xml) -# Structure is a comma separated list of structure -# , , - -declare -a files_to_validate_version=( - "infra/charts/feast/Chart.yaml,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-core/Chart.yaml,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-core/values.yaml,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-core/README.md,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-core/README.md,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-serving/Chart.yaml,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-jupyter/values.yaml,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-jupyter/README.md,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-jupyter/README.md,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-jupyter/Chart.yaml,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-serving/values.yaml,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/requirements.yaml,3,${FEAST_MASTER_VERSION}" - "infra/charts/feast/requirements.lock,3,${FEAST_MASTER_VERSION}" - "infra/docker-compose/.env.sample,1,${FEAST_RELEASE_VERSION}" - "datatypes/java/README.md,1,${FEAST_MASTER_VERSION}" - "docs/contributing/development-guide.md,4,${FEAST_MASTER_VERSION}" - "CHANGELOG.md,2,${FEAST_STABLE_VERSION}" -) - -echo -echo "Testing list of files to ensure they have the correct version" -echo - - -IS_LINT_SUCCESS=true -for i in "${files_to_validate_version[@]}"; do - IFS=',' read -r FILE_PATH EXPECTED_OCCURRENCES VERSION <<<"${i}" - # Disable version lint if '_ANY' specified as version. - if [ "$VERSION" = "_ANY" ] - then - continue - fi - - echo "=========================================================" - echo "Testing whether versions are correctly set within file: $FILE_PATH" - ACTUAL_OCCURRENCES=$(grep -c -P "\bv?$VERSION\b" "$FILE_PATH" || true) - - if [ "${ACTUAL_OCCURRENCES}" -ge "${EXPECTED_OCCURRENCES}" ]; then - echo "OK: Expecting $EXPECTED_OCCURRENCES occurrences of '$VERSION' in $FILE_PATH, and found $ACTUAL_OCCURRENCES" - else - echo "FAIL: Expecting $EXPECTED_OCCURRENCES occurrences of '$VERSION' in $FILE_PATH, but found $ACTUAL_OCCURRENCES" - IS_LINT_SUCCESS=false - fi -done - -if $IS_LINT_SUCCESS; then exit 0; else exit 1; fi diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 4feb2505b1c..00000000000 --- a/pom.xml +++ /dev/null @@ -1,877 +0,0 @@ - - - - 4.0.0 - - Feast - Feature Store for Machine Learning - ${github.url} - - dev.feast - feast-parent - ${revision} - pom - - - datatypes/java - storage/api - storage/connectors - core - serving - sdk/java - docs/coverage/java - common - common-test - - - - 0.10.0-SNAPSHOT - https://github.com/feast-dev/feast - - UTF-8 - UTF-8 - - 1.30.2 - 3.12.2 - 3.12.2 - 2.3.1.RELEASE - 5.2.7.RELEASE - 5.3.0.RELEASE - 2.9.0.RELEASE - 1.111.1 - 0.8.0 - 1.9.10 - 1.3 - 5.4.18.Final - 2.5.0 - 2.28.2 - - 0.26.0 - - 2.12.1 - 6.0.8 - 2.9.9 - 2.0.2 - 2.5.0.RELEASE - 1.18.12 - 1.8.4 - 2.8.6 - 1.5.24 - 3.14.7 - 3.10 - 2.3.1 - 1.3.2 - 2.0.1.Final - 2.8.0 - 0.20.0 - 6.1.2.Final - 1.6.6 - - - - ${maven.multiModuleProjectDirectory} - - false - false - feast.common.auth.providers.http.client - - - - Gojek - https://www.gojek.com - - - - - Feast Authors - ${github.url} - Gojek - https://www.gojek.com - - - - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - ${github.url} - scm:git:${github.url}.git - scm:git:git@github.com:feast-dev/feast.git - HEAD - - - - GitHub Issues - ${github.url}/issues - - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - - - - - com.google.cloud - google-cloud-bigquery - ${com.google.cloud.version} - - - com.google.cloud - google-cloud-storage - ${com.google.cloud.version} - - - - - com.google.cloud - google-cloud-nio - 0.83.0-alpha - - - - io.opencensus - opencensus-api - ${opencensus.version} - - - io.opencensus - opencensus-contrib-grpc-util - ${opencensus.version} - - - io.opencensus - opencensus-contrib-http-util - ${opencensus.version} - - - - - io.grpc - grpc-core - ${grpc.version} - - - io.grpc - grpc-api - ${grpc.version} - - - io.grpc - grpc-context - ${grpc.version} - - - io.grpc - grpc-all - ${grpc.version} - - - io.grpc - grpc-okhttp - ${grpc.version} - - - io.grpc - grpc-auth - ${grpc.version} - - - io.grpc - grpc-grpclb - ${grpc.version} - - - io.grpc - grpc-alts - ${grpc.version} - - - io.grpc - grpc-netty - ${grpc.version} - - - io.grpc - grpc-netty-shaded - ${grpc.version} - - - io.grpc - grpc-protobuf - ${grpc.version} - - - io.grpc - grpc-services - ${grpc.version} - - - io.grpc - grpc-stub - ${grpc.version} - - - io.grpc - grpc-testing - ${grpc.version} - test - - - - - io.swagger - swagger-annotations - ${swagger.core.version} - - - com.squareup.okhttp3 - okhttp - ${okhttp.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp.version} - - - io.springfox - springfox-swagger2 - ${springfox.version} - - - io.springfox - springfox-swagger-ui - ${springfox.version} - - - - - net.devh - grpc-server-spring-boot-starter - ${grpc.spring.boot.starter.version} - - - - - io.prometheus - simpleclient - ${io.prometheus.version} - - - io.prometheus - simpleclient_servlet - ${io.prometheus.version} - - - - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-resource-server - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - - - com.google.auth - google-auth-library-oauth2-http - ${google.auth.library.oauth2.http.version} - - - - - joda-time - joda-time - ${joda.time.version} - - - com.datadoghq - java-dogstatsd-client - 2.6.1 - - - com.google.guava - guava - 26.0-jre - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - com.google.protobuf - protobuf-java-util - ${protobuf.version} - - - org.projectlombok - lombok - ${lombok.version} - provided - - - com.google.auto.value - auto-value-annotations - ${auto.value.version} - - - com.google.auto.value - auto-value - ${auto.value.version} - - - com.google.code.gson - gson - ${gson.version} - - - io.gsonfire - gson-fire - ${gson.fire.version} - - - - com.github.kstyrc - embedded-redis - 0.6 - test - - - - - org.apache.kafka - kafka_2.12 - ${kafka.version} - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - org.hibernate - hibernate-core - ${hibernate.version} - - - org.hibernate.validator - hibernate-validator - ${org.hibernate.validator.version} - - - net.bytebuddy - byte-buddy - ${byte-buddy.version} - - - org.mockito - mockito-core - ${mockito.version} - test - - - org.springframework.boot - spring-boot-starter-web - ${spring.boot.version} - - - org.springframework.boot - spring-boot-starter-logging - - - - - org.apache.logging.log4j - log4j-api - ${log4jVersion} - - - org.apache.logging.log4j - log4j-core - ${log4jVersion} - - - org.apache.logging.log4j - log4j-jul - ${log4jVersion} - - - org.apache.logging.log4j - log4j-web - ${log4jVersion} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4jVersion} - - - - - org.springframework.boot - spring-boot-dependencies - ${spring.boot.version} - pom - import - - - com.squareup.okio - okio - 1.17.2 - - - javax.xml.bind - jaxb-api - ${javax.xml.bind.version} - - - javax.annotation - javax.annotation-api - ${javax.annotation.version} - - - javax.validation - validation-api - ${javax.validation.version} - - - - - - - - kr.motd.maven - os-maven-plugin - 1.6.2 - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - - com.diffplug.spotless - spotless-maven-plugin - 1.26.1 - - - - ${license.content} - - - 1.7 - - - - src/main/java/**/BatchLoadsWithResult.java - - - - - - ${license.content} - - - 2.7.2 - ${parent.basedir}/.scalafmt.conf - - - - - - - spotless-check - process-test-classes - - check - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - 8 - - - com.google.auto.value - auto-value - ${auto.value.version} - - - org.projectlombok - lombok - ${lombok.version} - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.0.0-M2 - - - org.codehaus.mojo - extra-enforcer-rules - 1.2 - - - - - valid-build-environment - - enforce - - - - - [3.6,4.0) - - - [11.0,) - - - - - - - consistent-dependency-versions - - enforce - - - - - - - - - no-snapshot-deps-at-release - - enforce - - - - - true - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M5 - - @{argLine} -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true - ${skipUTs} - - - - org.junit.vintage - junit-vintage-engine - 5.5.2 - - - - - maven-failsafe-plugin - 3.0.0-M5 - - - org.junit.jupiter - junit-jupiter-engine - 5.6.2 - - - - - integration-tests - - integration-test - verify - - - - - - ${groupId}:${artifactId} - - - ${project.build.outputDirectory} - - - - - org.springframework.boot - spring-boot-maven-plugin - - - true - - - - build-info - - build-info - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true - - ossrh - https://oss.sonatype.org/ - - true - - - - - org.codehaus.mojo - flatten-maven-plugin - 1.1.0 - - oss - - - - flatten - process-resources - - flatten - - - - flatten.clean - clean - - clean - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - --pinentry-mode - loopback - - - - ${gpg.passphrase} - - - - - - - - - - io.fabric8 - docker-maven-plugin - 0.20.1 - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - - -Xlint:all - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.1.1 - - - - org.apache.maven.shared - maven-dependency-analyzer - 1.11.1 - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.1 - - all - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - false - - - - org.jacoco - jacoco-maven-plugin - 0.8.5 - - - - prepare-agent - - - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring.boot.version} - - - - repackage - - - - - exec - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - - - - diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml deleted file mode 100644 index 6b2295b6b33..00000000000 --- a/sdk/java/pom.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - 4.0.0 - - Feast SDK for Java - SDK for registering, storing, and retrieving features - feast-sdk - - - dev.feast - feast-parent - ${revision} - ../.. - - - - - 5.5.2 - 2.28.2 - 0.33.0 - - - - - dev.feast - datatypes-java - ${project.version} - - - - dev.feast - feast-common - ${project.version} - - - - - io.grpc - grpc-netty-shaded - - - io.grpc - grpc-protobuf - - - io.grpc - grpc-stub - - - io.grpc - grpc-testing - - - com.google.protobuf - protobuf-java-util - - - com.google.protobuf - protobuf-java - - - - - io.opentracing.contrib - opentracing-grpc - 0.2.3 - - - io.opentracing - opentracing-api - ${opentracing.version} - - - io.opentracing - opentracing-noop - ${opentracing.version} - - - - - org.slf4j - slf4j-api - - - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-params - ${junit.version} - test - - - org.apache.commons - commons-lang3 - 3.6 - compile - - - org.mockito - mockito-core - ${mockito.version} - test - - - org.mockito - mockito-inline - ${mockito.version} - test - - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - true - - - - org.jacoco - jacoco-maven-plugin - - - - - diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java deleted file mode 100644 index 94836d88aa4..00000000000 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import io.grpc.CallCredentials; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; -import io.opentracing.contrib.grpc.TracingClientInterceptor; -import io.opentracing.util.GlobalTracer; -import java.io.File; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.net.ssl.SSLException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("WeakerAccess") -public class FeastClient implements AutoCloseable { - Logger logger = LoggerFactory.getLogger(FeastClient.class); - - private static final int CHANNEL_SHUTDOWN_TIMEOUT_SEC = 5; - - private final ManagedChannel channel; - private final ServingServiceBlockingStub stub; - - /** - * Create a client to access Feast Serving. - * - * @param host hostname or ip address of Feast serving GRPC server - * @param port port number of Feast serving GRPC server - * @return {@link FeastClient} - */ - public static FeastClient create(String host, int port) { - // configure client with no security config. - return FeastClient.createSecure(host, port, SecurityConfig.newBuilder().build()); - } - - /** - * Create a authenticated client that can access Feast serving with authentication enabled. - * Supports the {@link CallCredentials} in the {@link feast.common.auth.credentials} package. - * - * @param host hostname or ip address of Feast serving GRPC server - * @param port port number of Feast serving GRPC server - * @param securityConfig security options to configure the Feast client. See {@link - * SecurityConfig} for options. - * @return {@link FeastClient} - */ - public static FeastClient createSecure(String host, int port, SecurityConfig securityConfig) { - // Configure client TLS - ManagedChannel channel = null; - if (securityConfig.isTLSEnabled()) { - if (securityConfig.getCertificatePath().isPresent()) { - String certificatePath = securityConfig.getCertificatePath().get(); - // Use custom certificate for TLS - File certificateFile = new File(certificatePath); - try { - channel = - NettyChannelBuilder.forAddress(host, port) - .useTransportSecurity() - .sslContext(GrpcSslContexts.forClient().trustManager(certificateFile).build()) - .build(); - } catch (SSLException e) { - throw new IllegalArgumentException( - String.format("Invalid Certificate provided at path: %s", certificatePath), e); - } - } else { - // Use system certificates for TLS - channel = ManagedChannelBuilder.forAddress(host, port).useTransportSecurity().build(); - } - } else { - // Disable TLS - channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); - } - - return new FeastClient(channel, securityConfig.getCredentials()); - } - - /** - * Obtain info about Feast Serving. - * - * @return {@link GetFeastServingInfoResponse} containing Feast version, Serving type etc. - */ - public GetFeastServingInfoResponse getFeastServingInfo() { - return stub.getFeastServingInfo(GetFeastServingInfoRequest.newBuilder().build()); - } - - /** - * Get online features from Feast, without indicating project, will use `default`. - * - *

See {@link #getOnlineFeatures(List, List, String)} - * - * @param featureRefs list of string feature references to retrieve in the following format - * featureTable:feature, where 'featureTable' and 'feature' refer to the FeatureTable and - * Feature names respectively. Only the Feature name is required. - * @param rows list of {@link Row} to select the entities to retrieve the features for. - * @return list of {@link Row} containing retrieved data fields. - */ - public List getOnlineFeatures(List featureRefs, List rows) { - return getOnlineFeatures(featureRefs, rows, ""); - } - - /** - * Get online features from Feast. - * - *

Example of retrieving online features for the driver FeatureTable, with features driver_id - * and driver_name - * - * 

{@code
-   * FeastClient client = FeastClient.create("localhost", 6566);
-   * List requestedFeatureIds = Arrays.asList("driver:driver_id", "driver:driver_name");
-   * List requestedRows =
-   *         Arrays.asList(Row.create().set("driver_id", 123), Row.create().set("driver_id", 456));
-   * List retrievedFeatures = client.getOnlineFeatures(requestedFeatureIds, requestedRows);
-   * retrievedFeatures.forEach(System.out::println);
-   * }
- * - * @param featureRefs list of string feature references to retrieve in the following format - * featureTable:feature, where 'featureTable' and 'feature' refer to the FeatureTable and - * Feature names respectively. Only the Feature name is required. - * @param rows list of {@link Row} to select the entities to retrieve the features for - * @param project {@link String} Specifies the project override. If specified uses the project for - * retrieval. Overrides the projects set in Feature References if also specified. - * @return list of {@link Row} containing retrieved data fields. - */ - public List getOnlineFeatures(List featureRefs, List rows, String project) { - List features = RequestUtil.createFeatureRefs(featureRefs); - // build entity rows and collect entity references - HashSet entityRefs = new HashSet<>(); - List entityRows = - rows.stream() - .map( - row -> { - entityRefs.addAll(row.getFields().keySet()); - return EntityRow.newBuilder() - .setTimestamp(row.getEntityTimestamp()) - .putAllFields(row.getFields()) - .build(); - }) - .collect(Collectors.toList()); - - GetOnlineFeaturesResponse response = - stub.getOnlineFeaturesV2( - GetOnlineFeaturesRequestV2.newBuilder() - .addAllFeatures(features) - .addAllEntityRows(entityRows) - .setProject(project) - .build()); - - return response.getFieldValuesList().stream() - .map( - fieldValues -> { - Row row = Row.create(); - for (String fieldName : fieldValues.getFieldsMap().keySet()) { - row.set( - fieldName, - fieldValues.getFieldsMap().get(fieldName), - fieldValues.getStatusesMap().get(fieldName)); - } - return row; - }) - .collect(Collectors.toList()); - } - - protected FeastClient(ManagedChannel channel, Optional credentials) { - this.channel = channel; - TracingClientInterceptor tracingInterceptor = - TracingClientInterceptor.newBuilder().withTracer(GlobalTracer.get()).build(); - - ServingServiceBlockingStub servingStub = - ServingServiceGrpc.newBlockingStub(tracingInterceptor.intercept(channel)); - - if (credentials.isPresent()) { - servingStub = servingStub.withCallCredentials(credentials.get()); - } - - this.stub = servingStub; - } - - public void close() throws Exception { - if (channel != null) { - channel.shutdown().awaitTermination(CHANNEL_SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS); - } - } -} diff --git a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java deleted file mode 100644 index 69c8f9f737a..00000000000 --- a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import java.util.List; -import java.util.stream.Collectors; - -@SuppressWarnings("WeakerAccess") -public class RequestUtil { - - /** - * Create feature references protos from given string feature reference. - * - * @param featureRefStrings to create Feature Reference protos from - * @return List of parsed {@link FeatureReferenceV2} protos - */ - public static List createFeatureRefs(List featureRefStrings) { - if (featureRefStrings == null) { - throw new IllegalArgumentException("FeatureReferences cannot be null"); - } - - List featureRefs = - featureRefStrings.stream() - .map(refStr -> parseFeatureRef(refStr)) - .collect(Collectors.toList()); - - return featureRefs; - } - - /** - * Parse a feature reference proto builder from the given featureRefString - * - * @param featureRefString string feature reference to parse from. - * @return a parsed {@link FeatureReferenceV2} - */ - public static FeatureReferenceV2 parseFeatureRef(String featureRefString) { - featureRefString = featureRefString.trim(); - if (featureRefString.isEmpty()) { - throw new IllegalArgumentException("Cannot parse a empty feature reference"); - } - if (featureRefString.contains("/")) { - throw new IllegalArgumentException( - String.format( - "Unsupported feature reference: Specifying project in string" - + " Feature References is not longer supported: %s", - featureRefString)); - } - if (!featureRefString.contains(":")) { - throw new IllegalArgumentException( - String.format( - "Unsupported feature reference: %s - FeatureTable name and Feature name should be provided in string" - + " Feature References, in : format.", - featureRefString)); - } - - String[] featureReferenceParts = featureRefString.split(":"); - FeatureReferenceV2 featureRef = - FeatureReferenceV2.newBuilder() - .setFeatureTable(featureReferenceParts[0]) - .setName(featureReferenceParts[1]) - .build(); - - return featureRef; - } -} diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java deleted file mode 100644 index 51f820e320a..00000000000 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.Value.ValCase; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -@SuppressWarnings("UnusedReturnValue") -public class Row { - private Timestamp entity_timestamp; - private HashMap fields; - private HashMap fieldStatuses; - - public static Row create() { - Row row = new Row(); - row.entity_timestamp = Timestamps.fromMillis(System.currentTimeMillis()); - row.fields = new HashMap<>(); - row.fieldStatuses = new HashMap<>(); - return row; - } - - public Row setEntityTimestamp(Instant timestamp) { - entity_timestamp = Timestamps.fromMillis(timestamp.toEpochMilli()); - return this; - } - - public Timestamp getEntityTimestamp() { - return entity_timestamp; - } - - public Row setEntityTimestamp(String dateTime) { - entity_timestamp = Timestamps.fromMillis(Instant.parse(dateTime).toEpochMilli()); - return this; - } - - public Row set(String fieldName, Object value) { - return this.set(fieldName, value, FieldStatus.PRESENT); - } - - public Row set(String fieldName, Object value, FieldStatus status) { - String valueType = value.getClass().getCanonicalName(); - switch (valueType) { - case "java.lang.Integer": - fields.put(fieldName, Value.newBuilder().setInt32Val((int) value).build()); - break; - case "java.lang.Long": - fields.put(fieldName, Value.newBuilder().setInt64Val((long) value).build()); - break; - case "java.lang.Float": - fields.put(fieldName, Value.newBuilder().setFloatVal((float) value).build()); - break; - case "java.lang.Double": - fields.put(fieldName, Value.newBuilder().setDoubleVal((double) value).build()); - break; - case "java.lang.String": - fields.put(fieldName, Value.newBuilder().setStringVal((String) value).build()); - break; - case "byte[]": - fields.put( - fieldName, Value.newBuilder().setBytesVal(ByteString.copyFrom((byte[]) value)).build()); - break; - case "feast.proto.types.ValueProto.Value": - fields.put(fieldName, (Value) value); - break; - default: - throw new IllegalArgumentException( - String.format( - "Type '%s' is unsupported in Feast. Please use one of these value types: Integer, Long, Float, Double, String, byte[].", - valueType)); - } - - fieldStatuses.put(fieldName, status); - return this; - } - - public Map getFields() { - return fields; - } - - public Integer getInt(String fieldName) { - return getValue(fieldName).map(Value::getInt32Val).orElse(null); - } - - public Long getLong(String fieldName) { - return getValue(fieldName).map(Value::getInt64Val).orElse(null); - } - - public Float getFloat(String fieldName) { - return getValue(fieldName).map(Value::getFloatVal).orElse(null); - } - - public Double getDouble(String fieldName) { - return getValue(fieldName).map(Value::getDoubleVal).orElse(null); - } - - public String getString(String fieldName) { - return getValue(fieldName).map(Value::getStringVal).orElse(null); - } - - public byte[] getByte(String fieldName) { - return getValue(fieldName).map(Value::getBytesVal).map(ByteString::toByteArray).orElse(null); - } - - public Map getStatuses() { - return fieldStatuses; - } - - public FieldStatus getStatus(String fieldName) { - return fieldStatuses.get(fieldName); - } - - @Override - public String toString() { - List parts = new ArrayList<>(); - fields.forEach( - (key, value) -> - parts.add( - key - + ":" - + (value.getValCase().equals(ValCase.VAL_NOT_SET) - ? "NULL" - : value.toString().trim()))); - return String.join(", ", parts); - } - - private Optional getValue(String fieldName) { - if (!fields.containsKey(fieldName)) { - throw new IllegalArgumentException( - String.format("Row does not contain field '%s'", fieldName)); - } - Value value = fields.get(fieldName); - if (value.getValCase().equals(ValCase.VAL_NOT_SET)) { - return Optional.empty(); - } - return Optional.of(value); - } -} diff --git a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java b/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java deleted file mode 100644 index bd3b34bc12d..00000000000 --- a/sdk/java/src/main/java/com/gojek/feast/SecurityConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import com.google.auto.value.AutoValue; -import io.grpc.CallCredentials; -import java.util.Optional; - -/** SecurityConfig captures the security related configuration for FeastClient */ -@AutoValue -public abstract class SecurityConfig { - /** - * Enables authentication If specified, the call credentials used to provide credentials to - * authenticate with Feast. - */ - public abstract Optional getCredentials(); - - /** Whether to use TLS transport security is use when connecting to Feast. */ - public abstract boolean isTLSEnabled(); - - /** - * If specified and TLS is enabled, provides path to TLS certificate use the verify Service - * identity. - */ - public abstract Optional getCertificatePath(); - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setCredentials(Optional credentials); - - public abstract Builder setTLSEnabled(boolean isTLSEnabled); - - public abstract Builder setCertificatePath(Optional certificatePath); - - public abstract SecurityConfig build(); - } - - public static SecurityConfig.Builder newBuilder() { - return new AutoValue_SecurityConfig.Builder() - .setCredentials(Optional.empty()) - .setTLSEnabled(false) - .setCertificatePath(Optional.empty()); - } -} diff --git a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java deleted file mode 100644 index c458a06425f..00000000000 --- a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.AdditionalAnswers.delegatesTo; -import static org.mockito.Mockito.mock; - -import com.google.protobuf.Timestamp; -import feast.common.auth.credentials.JwtCallCredentials; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; -import feast.proto.types.ValueProto.Value; -import io.grpc.*; -import io.grpc.ServerCall.Listener; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.stub.StreamObserver; -import io.grpc.testing.GrpcCleanupRule; -import java.time.Instant; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -public class FeastClientTest { - private final String AUTH_TOKEN = "test token"; - - @Rule public GrpcCleanupRule grpcRule; - private AtomicBoolean isAuthenticated; - - private ServingServiceImplBase servingMock = - mock( - ServingServiceImplBase.class, - delegatesTo( - new ServingServiceImplBase() { - @Override - public void getOnlineFeaturesV2( - GetOnlineFeaturesRequestV2 request, - StreamObserver responseObserver) { - if (!request.equals(FeastClientTest.getFakeRequest())) { - responseObserver.onError(Status.FAILED_PRECONDITION.asRuntimeException()); - } - - responseObserver.onNext(FeastClientTest.getFakeResponse()); - responseObserver.onCompleted(); - } - })); - - // Mock Authentication interceptor will flag authenticated request by setting isAuthenticated to - // true. - private ServerInterceptor mockAuthInterceptor = - new ServerInterceptor() { - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - final Metadata.Key authorizationKey = - Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); - if (headers.containsKey(authorizationKey)) { - isAuthenticated.set(true); - } - return next.startCall(call, headers); - } - }; - - private FeastClient client; - private FeastClient authenticatedClient; - - @Before - public void setup() throws Exception { - this.grpcRule = new GrpcCleanupRule(); - this.isAuthenticated = new AtomicBoolean(false); - // setup fake serving service - String serverName = InProcessServerBuilder.generateName(); - this.grpcRule.register( - InProcessServerBuilder.forName(serverName) - .directExecutor() - .addService(this.servingMock) - .intercept(mockAuthInterceptor) - .build() - .start()); - - // setup test feast client target - ManagedChannel channel = - this.grpcRule.register( - InProcessChannelBuilder.forName(serverName).directExecutor().build()); - this.client = new FeastClient(channel, Optional.empty()); - this.authenticatedClient = - new FeastClient(channel, Optional.of(new JwtCallCredentials(AUTH_TOKEN))); - } - - @Test - public void shouldGetOnlineFeatures() { - shouldGetOnlineFeaturesWithClient(this.client); - } - - @Test - public void shouldAuthenticateAndGetOnlineFeatures() { - isAuthenticated.set(false); - shouldGetOnlineFeaturesWithClient(this.authenticatedClient); - assertEquals(isAuthenticated.get(), true); - } - - private void shouldGetOnlineFeaturesWithClient(FeastClient client) { - List rows = - client.getOnlineFeatures( - Arrays.asList("driver:name", "driver:rating", "driver:null_value"), - Arrays.asList( - Row.create().set("driver_id", 1).setEntityTimestamp(Instant.ofEpochSecond(100))), - "driver_project"); - - assertEquals( - rows.get(0).getFields(), - new HashMap() { - { - put("driver_id", intValue(1)); - put("driver:name", strValue("david")); - put("driver:rating", intValue(3)); - put("driver:null_value", Value.newBuilder().build()); - } - }); - assertEquals( - rows.get(0).getStatuses(), - new HashMap() { - { - put("driver_id", FieldStatus.PRESENT); - put("driver:name", FieldStatus.PRESENT); - put("driver:rating", FieldStatus.PRESENT); - put("driver:null_value", FieldStatus.NULL_VALUE); - } - }); - } - - private static GetOnlineFeaturesRequestV2 getFakeRequest() { - // setup mock serving service stub - return GetOnlineFeaturesRequestV2.newBuilder() - .addFeatures( - FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("name").build()) - .addFeatures( - FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("rating").build()) - .addFeatures( - FeatureReferenceV2.newBuilder().setFeatureTable("driver").setName("null_value").build()) - .addEntityRows( - EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("driver_id", intValue(1))) - .setProject("driver_project") - .build(); - } - - private static GetOnlineFeaturesResponse getFakeResponse() { - return GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("driver_id", intValue(1)) - .putStatuses("driver_id", FieldStatus.PRESENT) - .putFields("driver:name", strValue("david")) - .putStatuses("driver:name", FieldStatus.PRESENT) - .putFields("driver:rating", intValue(3)) - .putStatuses("driver:rating", FieldStatus.PRESENT) - .putFields("driver:null_value", Value.newBuilder().build()) - .putStatuses("driver:null_value", FieldStatus.NULL_VALUE) - .build()) - .build(); - } - - private static Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - - private static Value intValue(int val) { - return Value.newBuilder().setInt32Val(val).build(); - } -} diff --git a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java deleted file mode 100644 index 1592e20664d..00000000000 --- a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gojek.feast; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.TextFormat; -import feast.common.models.FeatureV2; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.NullSource; - -class RequestUtilTest { - - private static Stream provideValidFeatureRefs() { - return Stream.of( - Arguments.of( - Arrays.asList("driver:driver_id"), - Arrays.asList( - FeatureReferenceV2.newBuilder() - .setFeatureTable("driver") - .setName("driver_id") - .build()))); - } - - @ParameterizedTest - @MethodSource("provideValidFeatureRefs") - void createFeatureRefs_ShouldReturnFeaturesForValidFeatureRefs( - List input, List expected) { - List actual = RequestUtil.createFeatureRefs(input); - // Order of the actual and expected FeatureTables do no not matter - actual.sort(Comparator.comparing(FeatureReferenceV2::getName)); - expected.sort(Comparator.comparing(FeatureReferenceV2::getName)); - assertEquals(expected.size(), actual.size()); - for (int i = 0; i < expected.size(); i++) { - String expectedString = TextFormat.printer().printToString(expected.get(i)); - String actualString = TextFormat.printer().printToString(actual.get(i)); - assertEquals(expectedString, actualString); - } - } - - @ParameterizedTest - @MethodSource("provideValidFeatureRefs") - void renderFeatureRef_ShouldReturnFeatureRefString( - List expected, List input) { - input = input.stream().map(ref -> ref.toBuilder().build()).collect(Collectors.toList()); - List actual = - input.stream().map(ref -> FeatureV2.getFeatureStringRef(ref)).collect(Collectors.toList()); - assertEquals(expected.size(), actual.size()); - for (int i = 0; i < expected.size(); i++) { - assertEquals(expected.get(i), actual.get(i)); - } - } - - private static Stream provideInvalidFeatureRefs() { - return Stream.of(Arguments.of(ImmutableList.of("project/feature"))); - } - - private static Stream provideMissingFeatureTableFeatureRefs() { - return Stream.of(Arguments.of(ImmutableList.of("feature"))); - } - - @ParameterizedTest - @MethodSource("provideInvalidFeatureRefs") - void createFeatureRefs_ShouldThrowExceptionForProjectInFeatureRefs(List input) { - assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input)); - } - - @ParameterizedTest - @MethodSource("provideMissingFeatureTableFeatureRefs") - void createFeatureRefs_ShouldThrowExceptionForMissingFeatureTableInFeatureRefs( - List input) { - assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input)); - } - - @ParameterizedTest - @NullSource - void createFeatureRefs_ShouldThrowExceptionForNullFeatureRefs(List input) { - assertThrows(IllegalArgumentException.class, () -> RequestUtil.createFeatureRefs(input)); - } -} diff --git a/serving/.dockerignore b/serving/.dockerignore deleted file mode 100644 index 441c3d1a50c..00000000000 --- a/serving/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -*/.gradle/ -.idea/ -*/build/ \ No newline at end of file diff --git a/serving/.gitignore b/serving/.gitignore deleted file mode 100644 index 6c6b6d8d8f8..00000000000 --- a/serving/.gitignore +++ /dev/null @@ -1,37 +0,0 @@ -### Scratch files ### -scratch* - -### Local Environment ### -*local*.env - -### Gradle ### -.gradle -**/build/ -!gradle/wrapper/gradle-wrapper.jar -feast-serving.jar - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### IntelliJ IDEA ### -.idea -*.iws -*.iml -*.ipr -/out/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -## Feast Temporary Files ## -/temp/ \ No newline at end of file diff --git a/serving/README.md b/serving/README.md deleted file mode 100644 index ab530bb60df..00000000000 --- a/serving/README.md +++ /dev/null @@ -1,90 +0,0 @@ -### Getting Started Guide for Feast Serving Developers - -Pre-requisites: - -- [Maven](https://maven.apache.org/install.html) build tool version 3.6.x -- A running Feast Core instance -- A running Store instance e.g. local Redis Store instance - -From the Feast project root directory, run the following Maven command to start Feast Serving gRPC service running on port 6566 locally: - -```bash -# Assumptions: -# - Local Feast Core is running on localhost:6565 -# Uses configuration from serving/src/main/resources/application.yml -mvn -pl serving spring-boot:run -Dspring-boot.run.arguments=\ ---feast.core-host=localhost,\ ---feast.core-port=6565 -``` - -If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) installed, you can check that Feast Serving is running -``` -grpc_cli ls localhost:6566 -grpc_cli call localhost:6566 GetFeastServingVersion '' -grpc_cli call localhost:6566 GetFeastServingType '' -``` - -```bash -grpc_cli call localhost:6565 ApplyFeatureSet ' -feature_set { - name: "driver" - entities { - name: "driver_id" - value_type: STRING - } - features { - name: "city" - value_type: STRING - } - features { - name: "booking_completed_count" - value_type: INT64 - } - source { - type: KAFKA - kafka_source_config { - bootstrap_servers: "localhost:9092" - } - } -} -' - -grpc_cli call localhost:6565 GetFeatureSets ' -filter { - feature_set_name: "driver" -} -' - -grpc_cli call localhost:6566 GetBatchFeatures ' -feature_sets { - name: "driver" - feature_names: "booking_completed_count" - max_age { - seconds: 86400 - } -} -entity_dataset { - entity_names: "driver_id" - entity_dataset_rows { - entity_timestamp { - seconds: 1569873954 - } - } -} -' -``` - -``` -python3 < - - - 4.0.0 - - - dev.feast - feast-parent - ${revision} - - - feast-serving - Feast Serving - Feature serving API service - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - - - - - org.jacoco - jacoco-maven-plugin - - - - org.springframework.boot - spring-boot-maven-plugin - - false - - - - build-info - - build-info - - - - - - - - - - - dev.feast - datatypes-java - ${project.version} - - - - - dev.feast - feast-storage-api - ${project.version} - - - - dev.feast - feast-storage-connector-redis - ${project.version} - - - - dev.feast - feast-common - ${project.version} - - - - - org.slf4j - slf4j-api - - - - org.springframework.boot - spring-boot-configuration-processor - true - - - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-log4j2 - - - org.apache.logging.log4j - log4j-web - - - - org.springframework.boot - spring-boot-devtools - true - - - - - org.springframework.boot - spring-boot-starter-actuator - - - - - io.grpc - grpc-services - - - - io.grpc - grpc-stub - - - - com.google.protobuf - protobuf-java-util - - - - - com.google.guava - guava - - - - joda-time - joda-time - - - - io.jaegertracing - jaeger-client - 1.3.2 - - - io.opentracing - opentracing-api - 0.33.0 - - - io.opentracing - opentracing-noop - 0.33.0 - - - io.opentracing.contrib - opentracing-grpc - 0.2.3 - - - - - io.prometheus - simpleclient - 0.8.0 - - - - - io.prometheus - simpleclient_hotspot - 0.8.0 - - - - - io.prometheus - simpleclient_servlet - 0.8.0 - - - io.prometheus - simpleclient_spring_boot - 0.8.0 - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - - - - io.grpc - grpc-testing - - - - org.springframework.boot - spring-boot-starter-test - test - - - org.junit.vintage - junit-vintage-engine - - - - - - org.mockito - mockito-core - test - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.github.kstyrc - embedded-redis - test - - - jakarta.validation - jakarta.validation-api - ${jakarta.validation.api.version} - - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security.oauth - spring-security-oauth2 - ${spring.security.oauth2.version} - - - org.springframework.security - spring-security-oauth2-client - ${spring.security.version} - - - org.springframework.security - spring-security-web - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - - - net.devh - grpc-server-spring-boot-starter - ${grpc.spring.boot.starter.version} - - - com.nimbusds - nimbus-jose-jwt - 8.2.1 - - - org.springframework.security - spring-security-oauth2-core - ${spring.security.version} - - - org.testcontainers - testcontainers - 1.15.1 - test - - - org.testcontainers - junit-jupiter - 1.15.1 - test - - - org.awaitility - awaitility - 3.0.0 - test - - - sh.ory.keto - keto-client - 0.4.4-alpha.1 - test - - - dev.feast - feast-common-test - ${project.version} - test - - - com.squareup.okhttp - okhttp - 2.7.4 - test - - - - - - profile-local - - - !ci - - - - - - org.apache.maven.plugins - maven-resources-plugin - 3.1.0 - - - @ - - false - - - - - - - - src/main/resources - true - - - - - - diff --git a/serving/src/main/java/feast/serving/ServingApplication.java b/serving/src/main/java/feast/serving/ServingApplication.java deleted file mode 100644 index ab036d04d18..00000000000 --- a/serving/src/main/java/feast/serving/ServingApplication.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving; - -import feast.serving.config.FeastProperties; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; -import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; - -@SpringBootApplication( - exclude = { - DataSourceAutoConfiguration.class, - DataSourceTransactionManagerAutoConfiguration.class, - HibernateJpaAutoConfiguration.class - }) -@EnableConfigurationProperties(FeastProperties.class) -public class ServingApplication { - public static void main(String[] args) { - SpringApplication.run(ServingApplication.class, args); - } -} diff --git a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java deleted file mode 100644 index 2bc97439f38..00000000000 --- a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import java.util.concurrent.ScheduledExecutorService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextClosedEvent; -import org.springframework.stereotype.Component; - -@Component -public class ContextClosedHandler implements ApplicationListener { - - @Autowired ScheduledExecutorService executor; - - @Override - public void onApplicationEvent(ContextClosedEvent event) { - executor.shutdown(); - } -} diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java deleted file mode 100644 index bf048459cba..00000000000 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ /dev/null @@ -1,684 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -// Feast configuration properties that maps Feast configuration from default application.yml file to -// a Java object. -// https://www.baeldung.com/configuration-properties-in-spring-boot -// https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.credentials.CoreAuthenticationProperties; -import feast.common.logging.config.LoggingProperties; -import feast.proto.core.StoreProto; -import java.util.*; -import java.util.stream.Collectors; -import javax.annotation.PostConstruct; -import javax.validation.ConstraintViolation; -import javax.validation.ConstraintViolationException; -import javax.validation.Validation; -import javax.validation.Validator; -import javax.validation.ValidatorFactory; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import org.apache.logging.log4j.core.config.plugins.validation.constraints.ValidHost; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.info.BuildProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; - -/** Feast Serving properties. */ -@ComponentScan("feast.common.logging") -@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true) -public class FeastProperties { - - /** - * Instantiates a new Feast Serving properties. - * - * @param buildProperties the build properties - */ - @Autowired - public FeastProperties(BuildProperties buildProperties) { - setVersion(buildProperties.getVersion()); - } - - /** Instantiates a new Feast class. */ - public FeastProperties() {} - - /* Feast Serving build version */ - @NotBlank private String version = "unknown"; - - /* Feast Core host to connect to. */ - @ValidHost @NotBlank private String coreHost; - - /* Feast Core port to connect to. */ - @Positive private int coreGrpcPort; - - private CoreAuthenticationProperties coreAuthentication; - - public CoreAuthenticationProperties getCoreAuthentication() { - return coreAuthentication; - } - - public void setCoreAuthentication(CoreAuthenticationProperties coreAuthentication) { - this.coreAuthentication = coreAuthentication; - } - - /* Feast Core port to connect to. */ - @Positive private int coreCacheRefreshInterval; - - private SecurityProperties security; - - @Bean - SecurityProperties securityProperties() { - return this.getSecurity(); - } - - /** - * Getter for SecurityProperties - * - * @return Returns the {@link SecurityProperties} object. - */ - public SecurityProperties getSecurity() { - return security; - } - - /** - * Setter for SecurityProperties - * - * @param security :input {@link SecurityProperties} object - */ - public void setSecurity(SecurityProperties security) { - this.security = security; - } - - /** - * Finds and returns the active store - * - * @return Returns the {@link Store} model object - */ - public Store getActiveStore() { - for (Store store : getStores()) { - if (activeStore.equals(store.getName())) { - return store; - } - } - throw new RuntimeException( - String.format("Active store is misconfigured. Could not find store: %s.", activeStore)); - } - - /** - * Set the name of the active store found in the "stores" configuration list - * - * @param activeStore String name to active store - */ - public void setActiveStore(String activeStore) { - this.activeStore = activeStore; - } - - /** Name of the active store configuration (only one store can be active at a time). */ - @NotBlank private String activeStore; - - /** - * Collection of store configurations. The active store is selected by the "activeStore" field. - */ - private List stores = new ArrayList<>(); - - /* Job Store properties to retain state of async jobs. */ - private JobStoreProperties jobStore; - - /* Metric tracing properties. */ - private TracingProperties tracing; - - /* Feast Audit Logging properties */ - @NotNull private LoggingProperties logging; - - @Bean - LoggingProperties loggingProperties() { - return getLogging(); - } - - /** - * Gets Serving store configuration as a list of {@link Store}. - * - * @return List of stores objects - */ - public List getStores() { - return stores; - } - - /** - * Gets Feast Serving build version. - * - * @return the build version - */ - public String getVersion() { - return version; - } - - /** - * Sets build version - * - * @param version the build version - */ - public void setVersion(String version) { - this.version = version; - } - - /** - * Gets Feast Core host. - * - * @return Feast Core host - */ - public String getCoreHost() { - return coreHost; - } - - /** - * Sets Feast Core host to connect to. - * - * @param coreHost Feast Core host - */ - public void setCoreHost(String coreHost) { - this.coreHost = coreHost; - } - - /** - * Gets Feast Core gRPC port. - * - * @return Feast Core gRPC port - */ - public int getCoreGrpcPort() { - return coreGrpcPort; - } - - /** - * Sets Feast Core gRPC port. - * - * @param coreGrpcPort gRPC port of Feast Core - */ - public void setCoreGrpcPort(int coreGrpcPort) { - this.coreGrpcPort = coreGrpcPort; - } - - /** - * Gets CachedSpecService refresh interval. - * - * @return CachedSpecService refresh interval - */ - public int getCoreCacheRefreshInterval() { - return coreCacheRefreshInterval; - } - - /** - * Sets CachedSpecService refresh interval. - * - * @param coreCacheRefreshInterval CachedSpecService refresh interval - */ - public void setCoreCacheRefreshInterval(int coreCacheRefreshInterval) { - this.coreCacheRefreshInterval = coreCacheRefreshInterval; - } - - /** - * Sets the collection of configured stores. - * - * @param stores List of {@link Store} - */ - public void setStores(List stores) { - this.stores = stores; - } - - /** Store configuration class for database that this Feast Serving uses. */ - public static class Store { - - private String name; - - private String type; - - private Map config = new HashMap<>(); - - private List subscriptions = new ArrayList<>(); - - /** - * Gets name of this store. This is unique to this specific instance. - * - * @return the name of the store - */ - public String getName() { - return name; - } - - /** - * Sets the name of this store. - * - * @param name the name of the store - */ - public void setName(String name) { - this.name = name; - } - - /** - * Gets the store type. Example are REDIS or BIGQUERY - * - * @return the store type as a String. - */ - public String getType() { - return type; - } - - /** - * Sets the store type - * - * @param type the type - */ - public void setType(String type) { - this.type = type; - } - - /** - * Converts this {@link Store} to a {@link StoreProto.Store} - * - * @return {@link StoreProto.Store} with configuration set - * @throws InvalidProtocolBufferException the invalid protocol buffer exception - * @throws JsonProcessingException the json processing exception - */ - public StoreProto.Store toProto() - throws InvalidProtocolBufferException, JsonProcessingException { - List subscriptions = getSubscriptions(); - List subscriptionProtos = - subscriptions.stream().map(Subscription::toProto).collect(Collectors.toList()); - - StoreProto.Store.Builder storeProtoBuilder = - StoreProto.Store.newBuilder() - .setName(name) - .setType(StoreProto.Store.StoreType.valueOf(type)) - .addAllSubscriptions(subscriptionProtos); - - ObjectMapper jsonWriter = new ObjectMapper(); - - // TODO: All of this logic should be moved to the store layer. Only a Map - // should be sent to a store and it should do its own validation. - switch (StoreProto.Store.StoreType.valueOf(type)) { - case REDIS_CLUSTER: - StoreProto.Store.RedisClusterConfig.Builder redisClusterConfig = - StoreProto.Store.RedisClusterConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisClusterConfig); - return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig.build()).build(); - case REDIS: - StoreProto.Store.RedisConfig.Builder redisConfig = - StoreProto.Store.RedisConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisConfig); - return storeProtoBuilder.setRedisConfig(redisConfig.build()).build(); - default: - throw new InvalidProtocolBufferException("Invalid store set"); - } - } - - /** - * Get the subscriptions to this specific store. The subscriptions indicate which feature sets a - * store subscribes to. - * - * @return List of subscriptions. - */ - public List getSubscriptions() { - return subscriptions; - } - - /** - * Sets the store specific configuration. See getSubscriptions() for more details. - * - * @param subscriptions the subscriptions list - */ - public void setSubscriptions(List subscriptions) { - this.subscriptions = subscriptions; - } - - /** - * Gets the configuration to this specific store. This is a map of strings. These options are - * unique to the store. Please see protos/feast/core/Store.proto for the store specific - * configuration options - * - * @return Returns the store specific configuration - */ - public Map getConfig() { - return config; - } - - /** - * Sets the store config. Please protos/feast/core/Store.proto for the specific options for each - * store. - * - * @param config the config map - */ - public void setConfig(Map config) { - this.config = config; - } - - /** - * The Subscription type. - * - *

Note: Please see protos/feast/core/CoreService.proto for details on how to subscribe to - * feature sets. - */ - public static class Subscription { - /** Feast project to subscribe to. */ - String project; - - /** Feature set to subscribe to. */ - String name; - - /** Feature set versions to subscribe to. */ - String version; - - /** Project/Feature set exclude flag to subscribe to. */ - boolean exclude; - - /** - * Gets Feast project subscribed to. - * - * @return the project string - */ - public String getProject() { - return project; - } - - /** - * Sets Feast project to subscribe to for this store. - * - * @param project the project - */ - public void setProject(String project) { - this.project = project; - } - - /** - * Gets the feature set name to subscribe to. - * - * @return the name - */ - public String getName() { - return name; - } - - /** - * Sets the feature set name to subscribe to. - * - * @param name the name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Gets the feature set version that is being subscribed to by this store. - * - * @return the version - */ - public String getVersion() { - return version; - } - - /** - * Sets the feature set version that is being subscribed to by this store. - * - * @param version the version - */ - public void setVersion(String version) { - this.version = version; - } - - /** - * Gets the exclude flag to subscribe to. - * - * @return the exclude flag - */ - public boolean getExclude() { - return exclude; - } - - /** - * Sets the exclude flag to subscribe to. - * - * @param exclude the exclude flag - */ - public void setExclude(boolean exclude) { - this.exclude = exclude; - } - - /** - * Convert this {@link Subscription} to a {@link StoreProto.Store.Subscription}. - * - * @return the store proto . store . subscription - */ - public StoreProto.Store.Subscription toProto() { - return StoreProto.Store.Subscription.newBuilder() - .setName(getName()) - .setProject(getProject()) - .setExclude(getExclude()) - .build(); - } - } - } - - /** - * Gets job store properties - * - * @return the job store properties - */ - public JobStoreProperties getJobStore() { - return jobStore; - } - - /** - * Set job store properties - * - * @param jobStore Job store properties to set - */ - public void setJobStore(JobStoreProperties jobStore) { - this.jobStore = jobStore; - } - - /** - * Gets tracing properties - * - * @return tracing properties - */ - public TracingProperties getTracing() { - return tracing; - } - - /** - * Sets the tracing configuration. - * - * @param tracing the tracing - */ - public void setTracing(TracingProperties tracing) { - this.tracing = tracing; - } - - /** - * Gets logging properties - * - * @return logging properties - */ - public LoggingProperties getLogging() { - return logging; - } - - /** Sets logging properties @@param logging the logging properties */ - public void setLogging(LoggingProperties logging) { - this.logging = logging; - } - - /** The type Job store properties. */ - public static class JobStoreProperties { - - /** Job Store Redis Host */ - private String redisHost; - - /** Job Store Redis Host */ - private int redisPort; - - /** - * Gets redis host. - * - * @return the redis host - */ - public String getRedisHost() { - return redisHost; - } - - /** - * Sets redis host. - * - * @param redisHost the redis host - */ - public void setRedisHost(String redisHost) { - this.redisHost = redisHost; - } - - /** - * Gets redis port. - * - * @return the redis port - */ - public int getRedisPort() { - return redisPort; - } - - /** - * Sets redis port. - * - * @param redisPort the redis port - */ - public void setRedisPort(int redisPort) { - this.redisPort = redisPort; - } - } - - /** Trace metric collection properties */ - public static class TracingProperties { - - /** Tracing enabled/disabled */ - private boolean enabled; - - /** Name of tracer to use (only "jaeger") */ - private String tracerName; - - /** Service name uniquely identifies this Feast Serving deployment */ - private String serviceName; - - /** - * Is tracing enabled - * - * @return boolean flag - */ - public boolean isEnabled() { - return enabled; - } - - /** - * Sets tracing enabled or disabled. - * - * @param enabled flag - */ - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - /** - * Gets tracer name ('jaeger') - * - * @return the tracer name - */ - public String getTracerName() { - return tracerName; - } - - /** - * Sets tracer name. - * - * @param tracerName the tracer name - */ - public void setTracerName(String tracerName) { - this.tracerName = tracerName; - } - - /** - * Gets the service name. The service name uniquely identifies this Feast serving instance. - * - * @return the service name - */ - public String getServiceName() { - return serviceName; - } - - /** - * Sets service name. - * - * @param serviceName the service name - */ - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - } - - /** - * Validates all FeastProperties. This method runs after properties have been initialized and - * individually and conditionally validates each class. - */ - @PostConstruct - public void validate() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - - // Validate root fields in FeastProperties - Set> violations = validator.validate(this); - if (!violations.isEmpty()) { - throw new ConstraintViolationException(violations); - } - - // Validate CoreAuthenticationProperties - Set> coreAuthenticationPropsViolations = - validator.validate(getCoreAuthentication()); - if (!coreAuthenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(coreAuthenticationPropsViolations); - } - - // Validate AuthenticationProperties - Set> authenticationPropsViolations = - validator.validate(getSecurity().getAuthentication()); - if (!authenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authenticationPropsViolations); - } - - // Validate AuthorizationProperties - Set> authorizationPropsViolations = - validator.validate(getSecurity().getAuthorization()); - if (!authorizationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authorizationPropsViolations); - } - } -} diff --git a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java deleted file mode 100644 index 295b263f664..00000000000 --- a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import io.opentracing.Tracer; -import io.opentracing.contrib.grpc.TracingServerInterceptor; -import io.opentracing.noop.NoopTracerFactory; -import io.prometheus.client.exporter.MetricsServlet; -import io.prometheus.client.hotspot.DefaultExports; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.web.servlet.ServletRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class InstrumentationConfig { - - private FeastProperties feastProperties; - - @Autowired - public InstrumentationConfig(FeastProperties feastProperties) { - this.feastProperties = feastProperties; - } - - @Bean - public ServletRegistrationBean servletRegistrationBean() { - DefaultExports.initialize(); - return new ServletRegistrationBean(new MetricsServlet(), "/metrics"); - } - - @Bean - public Tracer tracer() { - if (!feastProperties.getTracing().isEnabled()) { - return NoopTracerFactory.create(); - } - - if (!feastProperties.getTracing().getTracerName().equalsIgnoreCase("jaeger")) { - throw new IllegalArgumentException("Only 'jaeger' tracer is supported for now."); - } - - return io.jaegertracing.Configuration.fromEnv(feastProperties.getTracing().getServiceName()) - .getTracer(); - } - - @Bean - public TracingServerInterceptor tracingInterceptor(Tracer tracer) { - return TracingServerInterceptor.newBuilder().withTracer(tracer).build(); - } -} diff --git a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java deleted file mode 100644 index ce4fe134373..00000000000 --- a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.http.converter.protobuf.ProtobufJsonFormatHttpMessageConverter; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class ServingApiConfiguration implements WebMvcConfigurer { - @Autowired private ProtobufJsonFormatHttpMessageConverter protobufConverter; - - @Bean - ProtobufJsonFormatHttpMessageConverter protobufHttpMessageConverter() { - return new ProtobufJsonFormatHttpMessageConverter(); - } - - @Override - public void configureMessageConverters(List> converters) { - converters.add(protobufConverter); - } -} diff --git a/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java b/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java deleted file mode 100644 index f51e06292da..00000000000 --- a/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import feast.common.auth.credentials.GoogleAuthCredentials; -import feast.common.auth.credentials.OAuthCredentials; -import feast.proto.serving.ServingServiceGrpc; -import io.grpc.CallCredentials; -import io.grpc.health.v1.HealthGrpc; -import java.io.IOException; -import net.devh.boot.grpc.server.security.check.AccessPredicate; -import net.devh.boot.grpc.server.security.check.GrpcSecurityMetadataSource; -import net.devh.boot.grpc.server.security.check.ManualGrpcSecurityMetadataSource; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -/* - * Copyright 2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -@Configuration -@ComponentScan( - basePackages = { - "feast.common.auth.config", - "feast.common.auth.service", - "feast.common.logging.interceptors" - }) -public class ServingSecurityConfig { - - private final FeastProperties feastProperties; - - public ServingSecurityConfig(FeastProperties feastProperties) { - this.feastProperties = feastProperties; - } - - /** - * Creates a SecurityMetadataSource when authentication is enabled. This allows for the - * configuration of endpoint level security rules. - * - * @return GrpcSecurityMetadataSource - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcSecurityMetadataSource grpcSecurityMetadataSource() { - final ManualGrpcSecurityMetadataSource source = new ManualGrpcSecurityMetadataSource(); - - // Authentication is enabled for all gRPC endpoints - source.setDefault(AccessPredicate.authenticated()); - - // The following endpoints allow unauthenticated access - source.set(ServingServiceGrpc.getGetFeastServingInfoMethod(), AccessPredicate.permitAll()); - source.set(HealthGrpc.getCheckMethod(), AccessPredicate.permitAll()); - return source; - } - - /** - * Creates a CallCredentials when authentication is enabled on core. This allows serving to - * connect to core with CallCredentials - * - * @return CallCredentials - */ - @Bean - @ConditionalOnProperty(prefix = "feast.core-authentication", name = "enabled") - CallCredentials CoreGrpcAuthenticationCredentials() throws IOException { - switch (feastProperties.getCoreAuthentication().getProvider()) { - case "google": - return new GoogleAuthCredentials(feastProperties.getCoreAuthentication().getOptions()); - case "oauth": - return new OAuthCredentials(feastProperties.getCoreAuthentication().getOptions()); - default: - throw new IllegalArgumentException( - "Please configure an Core Authentication Provider " - + "if you have enabled Authentication on core. " - + "Currently `google` and `oauth` are supported"); - } - } -} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java deleted file mode 100644 index 9ec35c33e19..00000000000 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; -import feast.serving.service.OnlineServingServiceV2; -import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.OnlineRetrieverV2; -import feast.storage.connectors.redis.retriever.*; -import io.opentracing.Tracer; -import org.slf4j.Logger; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ServingServiceConfigV2 { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); - - @Bean - public ServingServiceV2 servingServiceV2( - FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) - throws InvalidProtocolBufferException, JsonProcessingException { - ServingServiceV2 servingService = null; - FeastProperties.Store store = feastProperties.getActiveStore(); - StoreProto.Store.StoreType storeType = store.toProto().getType(); - - switch (storeType) { - case REDIS_CLUSTER: - RedisClientAdapter redisClusterClient = - RedisClusterClient.create(store.toProto().getRedisClusterConfig()); - OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient); - servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer); - break; - case REDIS: - RedisClientAdapter redisClient = RedisClient.create(store.toProto().getRedisConfig()); - OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient); - servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer); - break; - case UNRECOGNIZED: - case INVALID: - throw new IllegalArgumentException( - String.format( - "Unsupported store type '%s' for store name '%s'", - store.getType(), store.getName())); - } - - return servingService; - } -} diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java deleted file mode 100644 index b41a0f534ac..00000000000 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreSpecService; -import io.grpc.CallCredentials; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class SpecServiceConfig { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); - private String feastCoreHost; - private int feastCorePort; - private int feastCachedSpecServiceRefreshInterval; - - @Autowired - public SpecServiceConfig(FeastProperties feastProperties) { - this.feastCoreHost = feastProperties.getCoreHost(); - this.feastCorePort = feastProperties.getCoreGrpcPort(); - this.feastCachedSpecServiceRefreshInterval = feastProperties.getCoreCacheRefreshInterval(); - } - - @Bean - public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( - CachedSpecService cachedSpecStorage) { - ScheduledExecutorService scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(); - // reload all specs including new ones periodically - scheduledExecutorService.scheduleAtFixedRate( - cachedSpecStorage::scheduledPopulateCache, - feastCachedSpecServiceRefreshInterval, - feastCachedSpecServiceRefreshInterval, - TimeUnit.SECONDS); - return scheduledExecutorService; - } - - @Bean - public CachedSpecService specService( - FeastProperties feastProperties, ObjectProvider callCredentials) - throws InvalidProtocolBufferException, JsonProcessingException { - CoreSpecService coreService = - new CoreSpecService(feastCoreHost, feastCorePort, callCredentials); - StoreProto.Store storeProto = feastProperties.getActiveStore().toProto(); - CachedSpecService cachedSpecStorage = new CachedSpecService(coreService, storeProto); - try { - cachedSpecStorage.populateCache(); - } catch (Exception e) { - log.error("Unable to preload feast's spec"); - } - return cachedSpecStorage; - } -} diff --git a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java b/serving/src/main/java/feast/serving/config/WebSecurityConfig.java deleted file mode 100644 index f7b24a77405..00000000000 --- a/serving/src/main/java/feast/serving/config/WebSecurityConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; - -/** - * WebSecurityConfig disables auto configuration of Spring HTTP Security and allows security methods - * to be overridden - */ -@Configuration -@EnableWebSecurity -public class WebSecurityConfig extends WebSecurityConfigurerAdapter { - - /** - * Allows for custom web security rules to be applied. - * - * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception - */ - @Override - protected void configure(HttpSecurity http) throws Exception { - - // Bypasses security/authentication for the following paths - http.authorizeRequests() - .antMatchers("/actuator/**", "/metrics/**") - .permitAll() - .anyRequest() - .authenticated() - .and() - .csrf() - .disable(); - } -} diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java deleted file mode 100644 index 6615cf56eb6..00000000000 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.controller; - -import feast.proto.core.StoreProto.Store; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.serving.interceptors.GrpcMonitoringInterceptor; -import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; -import io.grpc.health.v1.HealthGrpc.HealthImplBase; -import io.grpc.health.v1.HealthProto.HealthCheckRequest; -import io.grpc.health.v1.HealthProto.HealthCheckResponse; -import io.grpc.health.v1.HealthProto.ServingStatus; -import io.grpc.stub.StreamObserver; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; - -// Reference: https://github.com/grpc/grpc/blob/master/doc/health-checking.md - -@GrpcService(interceptors = {GrpcMonitoringInterceptor.class}) -public class HealthServiceController extends HealthImplBase { - private CachedSpecService specService; - private ServingServiceV2 servingService; - - @Autowired - public HealthServiceController(CachedSpecService specService, ServingServiceV2 servingService) { - this.specService = specService; - this.servingService = servingService; - } - - @Override - public void check( - HealthCheckRequest request, StreamObserver responseObserver) { - // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g. - // if it's online service check that it the service can retrieve dummy/random - // feature table. - // Implement similary for batch service. - - try { - Store store = specService.getStore(); - servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance()); - responseObserver.onNext( - HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); - } catch (Exception e) { - responseObserver.onNext( - HealthCheckResponse.newBuilder().setStatus(ServingStatus.NOT_SERVING).build()); - } - responseObserver.onCompleted(); - } -} diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java deleted file mode 100644 index 531be39f9d6..00000000000 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.controller; - -import feast.common.auth.service.AuthorizationService; -import feast.common.logging.interceptors.GrpcMessageInterceptor; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; -import feast.serving.config.FeastProperties; -import feast.serving.exception.SpecRetrievalException; -import feast.serving.interceptors.GrpcMonitoringInterceptor; -import feast.serving.service.ServingServiceV2; -import feast.serving.util.RequestHelper; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.contrib.grpc.TracingServerInterceptor; -import net.devh.boot.grpc.server.service.GrpcService; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.context.SecurityContextHolder; - -@GrpcService( - interceptors = { - TracingServerInterceptor.class, - GrpcMessageInterceptor.class, - GrpcMonitoringInterceptor.class - }) -public class ServingServiceGRpcController extends ServingServiceImplBase { - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(ServingServiceGRpcController.class); - private final ServingServiceV2 servingServiceV2; - private final String version; - private final Tracer tracer; - private final AuthorizationService authorizationService; - - @Autowired - public ServingServiceGRpcController( - AuthorizationService authorizationService, - ServingServiceV2 servingServiceV2, - FeastProperties feastProperties, - Tracer tracer) { - this.authorizationService = authorizationService; - this.servingServiceV2 = servingServiceV2; - this.version = feastProperties.getVersion(); - this.tracer = tracer; - } - - @Override - public void getFeastServingInfo( - GetFeastServingInfoRequest request, - StreamObserver responseObserver) { - GetFeastServingInfoResponse feastServingInfo = servingServiceV2.getFeastServingInfo(request); - feastServingInfo = feastServingInfo.toBuilder().setVersion(version).build(); - responseObserver.onNext(feastServingInfo); - responseObserver.onCompleted(); - } - - @Override - public void getOnlineFeaturesV2( - ServingAPIProto.GetOnlineFeaturesRequestV2 request, - StreamObserver responseObserver) { - try { - // authorize for the project in request object. - if (request.getProject() != null && !request.getProject().isEmpty()) { - // project set at root level overrides the project set at feature table level - this.authorizationService.authorizeRequest( - SecurityContextHolder.getContext(), request.getProject()); - } - RequestHelper.validateOnlineRequest(request); - Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); - GetOnlineFeaturesResponse onlineFeatures = servingServiceV2.getOnlineFeatures(request); - if (span != null) { - span.finish(); - } - responseObserver.onNext(onlineFeatures); - responseObserver.onCompleted(); - } catch (SpecRetrievalException e) { - log.error("Failed to retrieve specs in SpecService", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing one of the projects in request")); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.warn("Failed to get Online Features", e); - responseObserver.onError(e); - } - } -} diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java b/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java deleted file mode 100644 index 8a198a82019..00000000000 --- a/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.controller; - -import static feast.serving.util.mappers.ResponseJSONMapper.mapGetOnlineFeaturesResponse; - -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.serving.config.FeastProperties; -import feast.serving.service.ServingServiceV2; -import feast.serving.util.RequestHelper; -import java.util.List; -import java.util.Map; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class ServingServiceRestController { - - private final ServingServiceV2 servingService; - private final String version; - - @Autowired - public ServingServiceRestController( - ServingServiceV2 servingService, FeastProperties feastProperties) { - this.servingService = servingService; - this.version = feastProperties.getVersion(); - } - - @RequestMapping(value = "/api/v1/info", produces = "application/json") - public GetFeastServingInfoResponse getInfo() { - GetFeastServingInfoResponse feastServingInfo = - servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance()); - return feastServingInfo.toBuilder().setVersion(version).build(); - } - - @RequestMapping( - value = "/api/v1/features/online", - produces = "application/json", - consumes = "application/json") - public List> getOnlineFeatures( - @RequestBody GetOnlineFeaturesRequestV2 request) { - RequestHelper.validateOnlineRequest(request); - GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request); - return mapGetOnlineFeaturesResponse(onlineFeatures); - } -} diff --git a/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java b/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java deleted file mode 100644 index fbcba969fa4..00000000000 --- a/serving/src/main/java/feast/serving/exception/SpecRetrievalException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.exception; - -/** Application-specific exception for any failure of retrieving feature/entity/storage spec. */ -public class SpecRetrievalException extends RuntimeException { - public SpecRetrievalException() { - super(); - } - - public SpecRetrievalException(String message) { - super(message); - } - - public SpecRetrievalException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java deleted file mode 100644 index bc7ed8997e3..00000000000 --- a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.interceptors; - -import feast.serving.util.Metrics; -import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCall.Listener; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; - -/** - * GrpcMonitoringInterceptor intercepts GRPC calls to provide request latency histogram metrics in - * the Prometheus client. - */ -public class GrpcMonitoringInterceptor implements ServerInterceptor { - - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - - long startCallMillis = System.currentTimeMillis(); - String fullMethodName = call.getMethodDescriptor().getFullMethodName(); - String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); - - return next.startCall( - new SimpleForwardingServerCall(call) { - @Override - public void close(Status status, Metadata trailers) { - Metrics.requestLatency - .labels(methodName) - .observe((System.currentTimeMillis() - startCallMillis) / 1000f); - Metrics.grpcRequestCount.labels(methodName, status.getCode().name()).inc(); - super.close(status, trailers); - } - }, - headers); - } -} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java deleted file mode 100644 index 70dd6f73878..00000000000 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ /dev/null @@ -1,368 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static feast.common.models.FeatureTable.getFeatureTableStringRef; - -import com.google.protobuf.Duration; -import feast.common.models.FeatureV2; -import feast.proto.serving.ServingAPIProto.FeastServingType; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.types.ValueProto; -import feast.serving.exception.SpecRetrievalException; -import feast.serving.specs.CachedSpecService; -import feast.serving.util.Metrics; -import feast.storage.api.retriever.Feature; -import feast.storage.api.retriever.OnlineRetrieverV2; -import io.grpc.Status; -import io.opentracing.Span; -import io.opentracing.Tracer; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.slf4j.Logger; - -public class OnlineServingServiceV2 implements ServingServiceV2 { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class); - private final CachedSpecService specService; - private final Tracer tracer; - private final OnlineRetrieverV2 retriever; - - private static final HashMap - TYPE_TO_VAL_CASE = - new HashMap<>() { - { - put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); - put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); - put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); - put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); - put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); - put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); - put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); - put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); - put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); - put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); - put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); - put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); - put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); - put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); - } - }; - - public OnlineServingServiceV2( - OnlineRetrieverV2 retriever, CachedSpecService specService, Tracer tracer) { - this.retriever = retriever; - this.specService = specService; - this.tracer = tracer; - } - - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) - .build(); - } - - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 request) { - String projectName = request.getProject(); - List featureReferences = request.getFeaturesList(); - - // Autofill default project if project is not specified - if (projectName.isEmpty()) { - projectName = "default"; - } - - List entityRows = request.getEntityRowsList(); - List> values = - entityRows.stream().map(r -> new HashMap<>(r.getFieldsMap())).collect(Collectors.toList()); - List> statuses = - entityRows.stream() - .map(r -> getMetadataMap(r.getFieldsMap(), false, false)) - .collect(Collectors.toList()); - - Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.setTag("entities", entityRows.size()); - storageRetrievalSpan.setTag("features", featureReferences.size()); - } - List> entityRowsFeatures = - retriever.getOnlineFeatures(projectName, entityRows, featureReferences); - if (storageRetrievalSpan != null) { - storageRetrievalSpan.finish(); - } - - if (entityRowsFeatures.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } - - String finalProjectName = projectName; - Map featureMaxAges = - featureReferences.stream() - .distinct() - .collect( - Collectors.toMap( - Function.identity(), - ref -> specService.getFeatureTableSpec(finalProjectName, ref).getMaxAge())); - - Map featureValueTypes = - featureReferences.stream() - .distinct() - .collect( - Collectors.toMap( - Function.identity(), - ref -> { - try { - return specService.getFeatureSpec(finalProjectName, ref).getValueType(); - } catch (SpecRetrievalException e) { - return ValueProto.ValueType.Enum.INVALID; - } - })); - - Span postProcessingSpan = tracer.buildSpan("postProcessing").start(); - - for (int i = 0; i < entityRows.size(); i++) { - GetOnlineFeaturesRequestV2.EntityRow entityRow = entityRows.get(i); - List curEntityRowFeatures = entityRowsFeatures.get(i); - - Map featureReferenceFeatureMap = - getFeatureRefFeatureMap(curEntityRowFeatures); - - Map rowValues = values.get(i); - Map rowStatuses = statuses.get(i); - - for (FeatureReferenceV2 featureReference : featureReferences) { - if (featureReferenceFeatureMap.containsKey(featureReference)) { - Feature feature = featureReferenceFeatureMap.get(featureReference); - - ValueProto.Value.ValCase valueCase = feature.getFeatureValue().getValCase(); - - boolean isMatchingFeatureSpec = - checkSameFeatureSpec(featureValueTypes.get(feature.getFeatureReference()), valueCase); - boolean isOutsideMaxAge = - checkOutsideMaxAge( - feature, entityRow, featureMaxAges.get(feature.getFeatureReference())); - - Map valueMap = - unpackValueMap(feature, isOutsideMaxAge, isMatchingFeatureSpec); - rowValues.putAll(valueMap); - - // Generate metadata for feature values and merge into entityFieldsMap - Map statusMap = - getMetadataMap(valueMap, !isMatchingFeatureSpec, isOutsideMaxAge); - rowStatuses.putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); - } else { - Map valueMap = - new HashMap<>() { - { - put( - FeatureV2.getFeatureStringRef(featureReference), - ValueProto.Value.newBuilder().build()); - } - }; - rowValues.putAll(valueMap); - - Map statusMap = - getMetadataMap(valueMap, true, false); - rowStatuses.putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); - } - } - } - - if (postProcessingSpan != null) { - postProcessingSpan.finish(); - } - - populateHistogramMetrics(entityRows, featureReferences, projectName); - populateFeatureCountMetrics(featureReferences, projectName); - - // Build response field values from entityValuesMap and entityStatusesMap - // Response field values should be in the same order as the entityRows provided by the user. - List fieldValuesList = - IntStream.range(0, entityRows.size()) - .mapToObj( - entityRowIdx -> - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(values.get(entityRowIdx)) - .putAllStatuses(statuses.get(entityRowIdx)) - .build()) - .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); - } - - private boolean checkSameFeatureSpec( - ValueProto.ValueType.Enum valueTypeEnum, ValueProto.Value.ValCase valueCase) { - if (valueTypeEnum.equals(ValueProto.ValueType.Enum.INVALID)) { - return false; - } - - if (valueCase.equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return true; - } - - return TYPE_TO_VAL_CASE.get(valueTypeEnum).equals(valueCase); - } - - private static Map getFeatureRefFeatureMap(List features) { - return features.stream() - .collect(Collectors.toMap(Feature::getFeatureReference, Function.identity())); - } - - /** - * Generate Field level Status metadata for the given valueMap. - * - * @param valueMap map of field name to value to generate metadata for. - * @param isNotFound whether the given valueMap represents values that were not found in the - * online retriever. - * @param isOutsideMaxAge whether the given valueMap contains values with age outside - * FeatureTable's max age. - * @return a 1:1 map keyed by field name containing field status metadata instead of values in the - * given valueMap. - */ - private static Map getMetadataMap( - Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { - return valueMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - es -> { - ValueProto.Value fieldValue = es.getValue(); - if (isNotFound) { - return GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND; - } else if (isOutsideMaxAge) { - return GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE; - } else if (fieldValue.getValCase().equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { - return GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE; - } - return GetOnlineFeaturesResponse.FieldStatus.PRESENT; - })); - } - - private static Map unpackValueMap( - Feature feature, boolean isOutsideMaxAge, boolean isMatchingFeatureSpec) { - Map valueMap = new HashMap<>(); - - if (!isOutsideMaxAge && isMatchingFeatureSpec) { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), feature.getFeatureValue()); - } else { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.getFeatureReference()), - ValueProto.Value.newBuilder().build()); - } - - return valueMap; - } - - /** - * Determine if the feature data in the given feature row is outside maxAge. Data is outside - * maxAge to be when the difference ingestion time set in feature row and the retrieval time set - * in entity row exceeds FeatureTable max age. - * - * @param feature contains the ingestion timing and feature data. - * @param entityRow contains the retrieval timing of when features are pulled. - * @param maxAge feature's max age. - */ - private static boolean checkOutsideMaxAge( - Feature feature, GetOnlineFeaturesRequestV2.EntityRow entityRow, Duration maxAge) { - - if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set - return false; - } - - long givenTimestamp = entityRow.getTimestamp().getSeconds(); - if (givenTimestamp == 0) { - givenTimestamp = System.currentTimeMillis() / 1000; - } - long timeDifference = givenTimestamp - feature.getEventTimestamp().getSeconds(); - return timeDifference > maxAge.getSeconds(); - } - - /** - * Populate histogram metrics that can be used for analysing online retrieval calls - * - * @param entityRows entity rows provided in request - * @param featureReferences feature references provided in request - * @param project project name provided in request - */ - private void populateHistogramMetrics( - List entityRows, - List featureReferences, - String project) { - Metrics.requestEntityCountDistribution - .labels(project) - .observe(Double.valueOf(entityRows.size())); - Metrics.requestFeatureCountDistribution - .labels(project) - .observe(Double.valueOf(featureReferences.size())); - - long countDistinctFeatureTables = - featureReferences.stream() - .map(featureReference -> getFeatureTableStringRef(project, featureReference)) - .distinct() - .count(); - Metrics.requestFeatureTableCountDistribution - .labels(project) - .observe(Double.valueOf(countDistinctFeatureTables)); - } - - /** - * Populate count metrics that can be used for analysing online retrieval calls - * - * @param statusMap Statuses of features which have been requested - * @param project Project where request for features was called from - */ - private void populateCountMetrics( - Map statusMap, String project) { - statusMap.forEach( - (featureRefString, status) -> { - if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) { - Metrics.notFoundKeyCount.labels(project, featureRefString).inc(); - } - if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) { - Metrics.staleKeyCount.labels(project, featureRefString).inc(); - } - }); - } - - private void populateFeatureCountMetrics( - List featureReferences, String project) { - featureReferences.forEach( - featureReference -> - Metrics.requestFeatureCount - .labels(project, FeatureV2.getFeatureStringRef(featureReference)) - .inc()); - } -} diff --git a/serving/src/main/java/feast/serving/service/ServingServiceV2.java b/serving/src/main/java/feast/serving/service/ServingServiceV2.java deleted file mode 100644 index 05acb31b78e..00000000000 --- a/serving/src/main/java/feast/serving/service/ServingServiceV2.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; - -public interface ServingServiceV2 { - /** - * Get information about the Feast serving deployment. - * - *

For Bigquery deployments, this includes the default job staging location to load - * intermediate files to. Otherwise, this method only returns the current Feast Serving backing - * store type. - * - * @param getFeastServingInfoRequest {@link ServingAPIProto.GetFeastServingInfoRequest} - * @return {@link ServingAPIProto.GetFeastServingInfoResponse} - */ - ServingAPIProto.GetFeastServingInfoResponse getFeastServingInfo( - ServingAPIProto.GetFeastServingInfoRequest getFeastServingInfoRequest); - - /** - * Get features from an online serving store, given a list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReferenceV2}s to retrieve, and list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s to join the - * retrieved values to. - * - *

Features can be queried across feature tables, but each {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} must contain all - * entities for all feature tables included in the request. - * - *

This request is fulfilled synchronously. - * - * @param getFeaturesRequest {@link GetOnlineFeaturesRequestV2} containing list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReferenceV2}s to retrieve and list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s to join the - * retrieved values to. - * @return {@link GetOnlineFeaturesResponse} with list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues} for each {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} supplied. - */ - GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 getFeaturesRequest); -} diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java deleted file mode 100644 index f54e08fc607..00000000000 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureTableProto.FeatureTable; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store; -import feast.proto.serving.ServingAPIProto; -import feast.serving.exception.SpecRetrievalException; -import io.grpc.StatusRuntimeException; -import io.prometheus.client.Gauge; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.slf4j.Logger; - -/** In-memory cache of specs hosted in Feast Core. */ -public class CachedSpecService { - - private static final int MAX_SPEC_COUNT = 1000; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(CachedSpecService.class); - private static final String DEFAULT_PROJECT_NAME = "default"; - - private final CoreSpecService coreService; - private Store store; - - private static Gauge cacheLastUpdated = - Gauge.build() - .name("cache_last_updated") - .subsystem("feast_serving") - .help("epoch time of the last time the cache was updated") - .register(); - - private static Gauge featureTablesCount = - Gauge.build() - .name("feature_table_count") - .subsystem("feast_serving") - .help("number of feature tables served by this instance") - .register(); - - private final LoadingCache, FeatureTableSpec> featureTableCache; - - private final LoadingCache< - ImmutablePair, FeatureProto.FeatureSpecV2> - featureCache; - - public CachedSpecService(CoreSpecService coreService, StoreProto.Store store) { - this.coreService = coreService; - this.store = coreService.registerStore(store); - - CacheLoader, FeatureTableSpec> featureTableCacheLoader = - CacheLoader.from(k -> retrieveSingleFeatureTable(k.getLeft(), k.getRight())); - featureTableCache = - CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureTableCacheLoader); - - CacheLoader< - ImmutablePair, FeatureProto.FeatureSpecV2> - featureCacheLoader = - CacheLoader.from(k -> retrieveSingleFeature(k.getLeft(), k.getRight())); - featureCache = CacheBuilder.newBuilder().build(featureCacheLoader); - } - - /** - * Get the current store configuration. - * - * @return StoreProto.Store store configuration for this serving instance - */ - public Store getStore() { - return this.store; - } - - /** - * Reload the store configuration from the given config path, then retrieve the necessary specs - * from core to preload the cache. - */ - public void populateCache() { - ImmutablePair< - HashMap, FeatureTableSpec>, - HashMap< - ImmutablePair, - FeatureProto.FeatureSpecV2>> - specs = getFeatureTableMap(); - - featureTableCache.invalidateAll(); - featureTableCache.putAll(specs.getLeft()); - - featureTablesCount.set(featureTableCache.size()); - - featureCache.invalidateAll(); - featureCache.putAll(specs.getRight()); - - cacheLastUpdated.set(System.currentTimeMillis()); - } - - public void scheduledPopulateCache() { - try { - populateCache(); - } catch (Exception e) { - log.warn("Error updating store configuration and specs: {}", e.getMessage()); - } - } - - /** - * Provides a map for easy retrieval of FeatureTable spec using FeatureTable reference - * - * @return Map in the format of - */ - private ImmutablePair< - HashMap, FeatureTableSpec>, - HashMap< - ImmutablePair, - FeatureProto.FeatureSpecV2>> - getFeatureTableMap() { - HashMap, FeatureTableSpec> featureTables = new HashMap<>(); - HashMap, FeatureProto.FeatureSpecV2> - features = new HashMap<>(); - - List projects = - coreService.listProjects(ListProjectsRequest.newBuilder().build()).getProjectsList(); - - for (String project : projects) { - try { - ListFeatureTablesResponse featureTablesResponse = - coreService.listFeatureTables( - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject(project)) - .build()); - Map featureRefSpecMap = - new HashMap<>(); - for (FeatureTable featureTable : featureTablesResponse.getTablesList()) { - FeatureTableSpec spec = featureTable.getSpec(); - featureTables.put(ImmutablePair.of(project, spec.getName()), spec); - - String featureTableName = spec.getName(); - List featureSpecs = spec.getFeaturesList(); - for (FeatureProto.FeatureSpecV2 featureSpec : featureSpecs) { - ServingAPIProto.FeatureReferenceV2 featureReference = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(featureTableName) - .setName(featureSpec.getName()) - .build(); - features.put(ImmutablePair.of(project, featureReference), featureSpec); - } - } - - } catch (StatusRuntimeException e) { - throw new RuntimeException( - String.format("Unable to retrieve specs matching project %s", project), e); - } - } - return ImmutablePair.of(featureTables, features); - } - - private FeatureTableSpec retrieveSingleFeatureTable(String projectName, String tableName) { - FeatureTable table = - coreService - .getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setProject(projectName) - .setName(tableName) - .build()) - .getTable(); - return table.getSpec(); - } - - private FeatureProto.FeatureSpecV2 retrieveSingleFeature( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureTableSpec featureTableSpec = - getFeatureTableSpec(projectName, featureReference); // don't stress core too much - if (featureTableSpec == null) { - return null; - } - return featureTableSpec.getFeaturesList().stream() - .filter(f -> f.getName().equals(featureReference.getName())) - .findFirst() - .orElse(null); - } - - public FeatureTableSpec getFeatureTableSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureTableSpec featureTableSpec; - try { - featureTableSpec = - featureTableCache.get(ImmutablePair.of(projectName, featureReference.getFeatureTable())); - } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { - throw new SpecRetrievalException( - String.format( - "Unable to find FeatureTable %s/%s", projectName, featureReference.getFeatureTable()), - e); - } - - return featureTableSpec; - } - - public FeatureProto.FeatureSpecV2 getFeatureSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureProto.FeatureSpecV2 featureSpec; - try { - featureSpec = featureCache.get(ImmutablePair.of(projectName, featureReference)); - } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { - throw new SpecRetrievalException( - String.format("Unable to find Feature with name: %s", featureReference.getName()), e); - } - - return featureSpec; - } -} diff --git a/serving/src/main/java/feast/serving/specs/CoreSpecService.java b/serving/src/main/java/feast/serving/specs/CoreSpecService.java deleted file mode 100644 index 5429d229313..00000000000 --- a/serving/src/main/java/feast/serving/specs/CoreSpecService.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import feast.proto.core.StoreProto.Store; -import io.grpc.CallCredentials; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import org.slf4j.Logger; -import org.springframework.beans.factory.ObjectProvider; - -/** Client for interfacing with specs in Feast Core. */ -public class CoreSpecService { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(CoreSpecService.class); - private final CoreServiceGrpc.CoreServiceBlockingStub blockingStub; - - public CoreSpecService( - String feastCoreHost, int feastCorePort, ObjectProvider callCredentials) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress(feastCoreHost, feastCorePort).usePlaintext().build(); - CallCredentials creds = callCredentials.getIfAvailable(); - if (creds != null) { - blockingStub = CoreServiceGrpc.newBlockingStub(channel).withCallCredentials(creds); - } else { - blockingStub = CoreServiceGrpc.newBlockingStub(channel); - } - } - - public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) { - return blockingStub.updateStore(updateStoreRequest); - } - - /** - * Register the given store entry in Feast Core. If store already exists in Feast Core, updates - * the store entry in feast core. - * - * @param store entry to register/update in Feast Core. - * @return The register/updated store entry - */ - public Store registerStore(Store store) { - UpdateStoreRequest request = UpdateStoreRequest.newBuilder().setStore(store).build(); - try { - UpdateStoreResponse updateStoreResponse = this.updateStore(request); - if (!updateStoreResponse.getStore().equals(store)) { - throw new RuntimeException("Core store config not matching current store config"); - } - return updateStoreResponse.getStore(); - } catch (Exception e) { - throw new RuntimeException("Unable to update store configuration", e); - } - } - - public ListProjectsResponse listProjects(ListProjectsRequest listProjectsRequest) { - return blockingStub.listProjects(listProjectsRequest); - } - - public ListFeatureTablesResponse listFeatureTables( - ListFeatureTablesRequest listFeatureTablesRequest) { - return blockingStub.listFeatureTables(listFeatureTablesRequest); - } - - public CoreServiceProto.GetFeatureTableResponse getFeatureTable( - CoreServiceProto.GetFeatureTableRequest getFeatureTableRequest) { - return blockingStub.getFeatureTable(getFeatureTableRequest); - } -} diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java deleted file mode 100644 index 90b94930e71..00000000000 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.util; - -import io.prometheus.client.Counter; -import io.prometheus.client.Histogram; - -public class Metrics { - - public static final Histogram requestLatency = - Histogram.build() - .name("request_latency_seconds") - .subsystem("feast_serving") - .help("Request latency in seconds") - .labelNames("method") - .register(); - - public static final Histogram requestEntityCountDistribution = - Histogram.build() - .buckets(1, 2, 5, 10, 20, 50, 100, 200) - .name("request_entity_count_distribution") - .subsystem("feast_serving") - .help("Number of entity rows per request") - .labelNames("project") - .register(); - - public static final Histogram requestFeatureCountDistribution = - Histogram.build() - .buckets(1, 2, 5, 10, 15, 20, 30, 50) - .name("request_feature_count_distribution") - .subsystem("feast_serving") - .help("Number of feature rows per request") - .labelNames("project") - .register(); - - public static final Histogram requestFeatureTableCountDistribution = - Histogram.build() - .buckets(1, 2, 5, 10, 20) - .name("request_feature_table_count_distribution") - .subsystem("feast_serving") - .help("Number of feature tables per request") - .labelNames("project") - .register(); - - public static final Counter requestFeatureCount = - Counter.build() - .name("request_feature_count") - .subsystem("feast_serving") - .help("number of feature rows requested") - .labelNames("project", "feature_name") - .register(); - - public static final Counter notFoundKeyCount = - Counter.build() - .name("not_found_feature_count") - .subsystem("feast_serving") - .help("number requested feature rows that were not found") - .labelNames("project", "feature_name") - .register(); - - public static final Counter staleKeyCount = - Counter.build() - .name("stale_feature_count") - .subsystem("feast_serving") - .help("number requested feature rows that were stale") - .labelNames("project", "feature_name") - .register(); - - public static final Counter grpcRequestCount = - Counter.build() - .name("grpc_request_count") - .subsystem("feast_serving") - .help("number of grpc requests served") - .labelNames("method", "status_code") - .register(); -} diff --git a/serving/src/main/java/feast/serving/util/RequestHelper.java b/serving/src/main/java/feast/serving/util/RequestHelper.java deleted file mode 100644 index 4d478f430f2..00000000000 --- a/serving/src/main/java/feast/serving/util/RequestHelper.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.util; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; - -public class RequestHelper { - - public static void validateOnlineRequest(GetOnlineFeaturesRequestV2 request) { - // All EntityRows should not be empty - if (request.getEntityRowsCount() <= 0) { - throw new IllegalArgumentException("Entity value must be provided"); - } - // All FeatureReferences should have FeatureTable name and Feature name - for (FeatureReferenceV2 featureReference : request.getFeaturesList()) { - validateOnlineRequestFeatureReference(featureReference); - } - } - - public static void validateOnlineRequestFeatureReference(FeatureReferenceV2 featureReference) { - if (featureReference.getFeatureTable().isEmpty()) { - throw new IllegalArgumentException("FeatureTable name must be provided in FeatureReference"); - } - if (featureReference.getName().isEmpty()) { - throw new IllegalArgumentException("Feature name must be provided in FeatureReference"); - } - } -} diff --git a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java deleted file mode 100644 index 238df549513..00000000000 --- a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.util.mappers; - -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.ValueProto.Value; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -// ResponseJSONMapper maps GRPC Response types to more human readable JSON responses -public class ResponseJSONMapper { - - public static List> mapGetOnlineFeaturesResponse( - GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(fieldValues -> convertFieldValuesToMap(fieldValues)) - .collect(Collectors.toList()); - } - - private static Map convertFieldValuesToMap(FieldValues fieldValues) { - return fieldValues.getFieldsMap().entrySet().stream() - .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue()))); - } - - private static Object extractValue(Value value) { - switch (value.getValCase().getNumber()) { - case 1: - return value.getBytesVal(); - case 2: - return value.getStringVal(); - case 3: - return value.getInt32Val(); - case 4: - return value.getInt64Val(); - case 5: - return value.getDoubleVal(); - case 6: - return value.getFloatVal(); - case 7: - return value.getBoolVal(); - case 11: - return value.getBytesListVal(); - case 12: - return value.getStringListVal(); - case 13: - return value.getInt32ListVal(); - case 14: - return value.getInt64ListVal(); - case 15: - return value.getDoubleListVal(); - case 16: - return value.getFloatListVal(); - case 17: - return value.getBoolListVal(); - default: - return null; - } - } -} diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml deleted file mode 100644 index 288ec7eb972..00000000000 --- a/serving/src/main/resources/application.yml +++ /dev/null @@ -1,105 +0,0 @@ -feast: - # GRPC service address for Feast Core - # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) - core-host: ${FEAST_CORE_HOST:localhost} - core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - - core-authentication: - enabled: false # should be set to true if authentication is enabled on core. - provider: google # can be set to `oauth` or `google` - # if google, GOOGLE_APPLICATION_CREDENTIALS environment variable should be set. - options: - #if provider is oauth following properties need to be set, else serving boot up will fail. - oauth_url: https://localhost/oauth/token #oauth token request url - grant_type: client_credentials #oauth grant type - client_id: #oauth client id which will be used for jwt token token request - client_secret: #oauth client secret which will be used for jwt token token request - audience: https://localhost #token audience. - jwkEndpointURI: #jwk enpoint uri, used for caching token till expiry. - - core-cache-refresh-interval: 10 - - # Indicates the active store. Only a single store in the last can be active at one time. In the future this key - # will be deprecated in order to allow multiple stores to be served from a single serving instance - active_store: online - - security: - authentication: - enabled: false - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - subjectClaim: email - - authorization: - enabled: false - provider: http - options: - basePath: http://localhost:3000 - - # List of store configurations - stores: - # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options - - name: online # Name of the store (referenced by active_store) - type: REDIS # Type of the store. REDIS, REDIS_CLUSTER, BIGQUERY are available options - config: # Store specific configuration. See - host: localhost - port: 6379 - # Subscriptions indicate which feature sets needs to be retrieved and used to populate this store - subscriptions: - # Wildcards match all options. No filtering is done. - - name: "*" - project: "*" - - name: online_cluster - type: REDIS_CLUSTER - config: # Store specific configuration. - # Connection string specifies the host:port of Redis instances in the redis cluster. - connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" - read_from: MASTER - subscriptions: - - name: "*" - project: "*" - version: "*" - - tracing: - # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls - # which can be useful to debug performance issues and perform benchmarking - enabled: false - # Only Jaeger tracer is supported currently - # https://opentracing.io/docs/supported-tracers/ - tracer-name: jaeger - # The service name identifier for the tracing data - service-name: feast_serving - - logging: - # Audit logging provides a machine readable structured JSON log that can give better - # insight into what is happening in Feast. - audit: - # Whether audit logging is enabled. - enabled: true - # Whether to enable message level (ie request/response) audit logging - messageLogging: - enabled: false - # Logging forwarder currently provides a machine readable structured JSON log to an - # external fluentd service that can give better insight into what is happening in Feast. - # Accepts console / fluentd as destination - destination: console - fluentdHost: localhost - fluentdPort: 24224 - -grpc: - server: - # The port number Feast Serving GRPC service should listen on - # It is set default to 6566 so it does not conflict with the GRPC server on Feast Core - # which defaults to port 6565 - port: ${GRPC_PORT:6566} - security: - enabled: false - certificateChain: server.crt - privateKey: server.key - -server: - # The port number on which the Tomcat webserver that serves REST API endpoints should listen - # It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core - # if both Feast Core and Serving are running on the same machine - port: ${SERVER_PORT:8081} diff --git a/serving/src/main/resources/banner.txt b/serving/src/main/resources/banner.txt deleted file mode 100644 index 44aa9c4cad9..00000000000 --- a/serving/src/main/resources/banner.txt +++ /dev/null @@ -1,14 +0,0 @@ - -███████╗███████╗ █████╗ ███████╗████████╗ -██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝ -█████╗ █████╗ ███████║███████╗ ██║ -██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║ -██║ ███████╗██║ ██║███████║ ██║ -╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ - -███████╗███████╗██████╗ ██╗ ██╗██╗███╗ ██╗ ██████╗ -██╔════╝██╔════╝██╔══██╗██║ ██║██║████╗ ██║██╔════╝ -███████╗█████╗ ██████╔╝██║ ██║██║██╔██╗ ██║██║ ███╗ -╚════██║██╔══╝ ██╔══██╗╚██╗ ██╔╝██║██║╚██╗██║██║ ██║ -███████║███████╗██║ ██║ ╚████╔╝ ██║██║ ╚████║╚██████╔╝ -╚══════╝╚══════╝╚═╝ ╚═╝ ╚═══╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝ diff --git a/serving/src/main/resources/log4j2.xml b/serving/src/main/resources/log4j2.xml deleted file mode 100644 index c75c2db13cc..00000000000 --- a/serving/src/main/resources/log4j2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - diff --git a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java deleted file mode 100644 index c0901c48bcd..00000000000 --- a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.controller; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.protobuf.Timestamp; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.service.AuthorizationService; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.types.ValueProto.Value; -import feast.serving.config.FeastProperties; -import feast.serving.service.ServingServiceV2; -import io.grpc.stub.StreamObserver; -import io.jaegertracing.Configuration; -import io.opentracing.Tracer; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; - -public class ServingServiceGRpcControllerTest { - - @Mock private ServingServiceV2 mockServingServiceV2; - - @Mock private StreamObserver mockStreamObserver; - - private GetOnlineFeaturesRequestV2 validRequest; - - private ServingServiceGRpcController service; - - @Mock private Authentication authentication; - - @Mock private AuthorizationProvider authProvider; - - @Before - public void setUp() { - initMocks(this); - - validRequest = - GetOnlineFeaturesRequestV2.newBuilder() - .addFeatures( - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature1") - .build()) - .addFeatures( - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature2") - .build()) - .addEntityRows( - EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", Value.newBuilder().setInt64Val(1).build()) - .putFields("entity2", Value.newBuilder().setInt64Val(1).build())) - .build(); - } - - private ServingServiceGRpcController getServingServiceGRpcController(boolean enableAuth) { - Tracer tracer = Configuration.fromEnv("dummy").getTracer(); - FeastProperties feastProperties = new FeastProperties(); - - AuthorizationProperties authorizationProps = new AuthorizationProperties(); - authorizationProps.setEnabled(enableAuth); - AuthenticationProperties authenticationProps = new AuthenticationProperties(); - authenticationProps.setEnabled(enableAuth); - SecurityProperties securityProperties = new SecurityProperties(); - securityProperties.setAuthentication(authenticationProps); - securityProperties.setAuthorization(authorizationProps); - feastProperties.setSecurity(securityProperties); - AuthorizationService authorizationservice = - new AuthorizationService(feastProperties.getSecurity(), authProvider); - return new ServingServiceGRpcController( - authorizationservice, mockServingServiceV2, feastProperties, tracer); - } - - @Test - public void shouldPassValidRequestAsIs() { - service = getServingServiceGRpcController(false); - service.getOnlineFeaturesV2(validRequest, mockStreamObserver); - Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); - } - - @Test - public void shouldCallOnErrorIfEntityDatasetIsNotSet() { - service = getServingServiceGRpcController(false); - GetOnlineFeaturesRequestV2 missingEntityName = - GetOnlineFeaturesRequestV2.newBuilder(validRequest).clearEntityRows().build(); - service.getOnlineFeaturesV2(missingEntityName, mockStreamObserver); - Mockito.verify(mockStreamObserver).onError(Mockito.any(IllegalArgumentException.class)); - } - - @Test - public void shouldPassValidRequestAsIsIfRequestIsAuthorized() { - service = getServingServiceGRpcController(true); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(authentication); - doReturn(AuthorizationResult.success()) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeaturesV2(validRequest, mockStreamObserver); - Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); - } -} diff --git a/serving/src/test/java/feast/serving/it/AuthTestUtils.java b/serving/src/test/java/feast/serving/it/AuthTestUtils.java deleted file mode 100644 index a4c3db7defe..00000000000 --- a/serving/src/test/java/feast/serving/it/AuthTestUtils.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.protobuf.Timestamp; -import feast.common.auth.credentials.OAuthCredentials; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto.Value; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.junit.runners.model.InitializationError; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -public class AuthTestUtils { - - private static final String DEFAULT_FLAVOR = "glob"; - - public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( - String projectName, - String featureTableName, - String featureName, - String entityId, - int entityValue) { - return GetOnlineFeaturesRequestV2.newBuilder() - .setProject(projectName) - .addFeatures( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(featureTableName) - .setName(featureName) - .build()) - .addEntityRows( - GetOnlineFeaturesRequestV2.EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields(entityId, Value.newBuilder().setInt64Val(entityValue).build())) - .build(); - } - - public static CoreSimpleAPIClient getSecureApiClientForCore( - int feastCorePort, Map options) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feastCorePort).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new CoreSimpleAPIClient(secureCoreService); - } - - public static ServingServiceGrpc.ServingServiceBlockingStub getServingServiceStub( - boolean isSecure, int feastServingPort, Map options) { - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feastServingPort).usePlaintext().build(); - - if (isSecure) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - return ServingServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - } else { - return ServingServiceGrpc.newBlockingStub(secureChannel); - } - } - - public static void seedHydra( - String hydraExternalUrl, - String clientId, - String clientSecrret, - String audience, - String grantType) - throws IOException, InitializationError { - - OkHttpClient httpClient = new OkHttpClient(); - String createClientEndpoint = String.format("%s/%s", hydraExternalUrl, "clients"); - JsonObject jsonObject = new JsonObject(); - JsonArray audienceArrray = new JsonArray(); - audienceArrray.add(audience); - JsonArray grantTypes = new JsonArray(); - grantTypes.add(grantType); - jsonObject.addProperty("client_id", clientId); - jsonObject.addProperty("client_secret", clientSecrret); - jsonObject.addProperty("token_endpoint_auth_method", "client_secret_post"); - jsonObject.add("audience", audienceArrray); - jsonObject.add("grant_types", grantTypes); - MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - - RequestBody requestBody = RequestBody.create(JSON, jsonObject.toString()); - Request request = - new Request.Builder() - .url(createClientEndpoint) - .addHeader("Content-Type", "application/json") - .post(requestBody) - .build(); - Response response = httpClient.newCall(request).execute(); - if (!response.isSuccessful()) { - throw new InitializationError(response.message()); - } - } - - public static void seedKeto(String url, String project, String subjectInProject, String admin) - throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(project); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(admin); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = - getMyProjectMemberPolicyRole(project, subjectInProject); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole( - String project, String subjectInProject) { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole(String subjectIsAdmin) { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy(String project) { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } -} diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java deleted file mode 100644 index 79d47737e71..00000000000 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -@ActiveProfiles("it") -@SpringBootTest -public class BaseAuthIT { - - static final String FEATURE_TABLE_NAME = "featuretable_1"; - static final String FEATURE_NAME = "feature_1"; - static final String ENTITY_ID = "entity_id"; - static final String PROJECT_NAME = "project_1"; - static final int SERVICE_START_MAX_WAIT_TIME_IN_MINUTES = 3; - static final String CLIENT_ID = "client_id"; - static final String CLIENT_SECRET = "client_secret"; - static final String TOKEN_URL = "http://localhost:4444/oauth2/token"; - static final String JWK_URI = "http://localhost:4444/.well-known/jwks.json"; - - static final String GRANT_TYPE = "client_credentials"; - - static final String AUDIENCE = "https://localhost"; - - static final String CORE = "core_1"; - - static final String HYDRA = "hydra_1"; - static final int HYDRA_PORT = 4445; - - static CoreSimpleAPIClient insecureApiClient; - - static final String REDIS = "redis_1"; - static final int REDIS_PORT = 6379; - - static final int FEAST_CORE_PORT = 6565; - - @DynamicPropertySource - static void properties(DynamicPropertyRegistry registry) { - registry.add("feast.stores[0].name", () -> "online"); - registry.add("feast.stores[0].type", () -> "REDIS"); - // Redis needs to accessible by both core and serving, hence using host address - registry.add( - "feast.stores[0].config.host", - () -> { - try { - return InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - e.printStackTrace(); - return ""; - } - }); - registry.add("feast.stores[0].config.port", () -> REDIS_PORT); - registry.add("feast.stores[0].subscriptions[0].name", () -> "*"); - registry.add("feast.stores[0].subscriptions[0].project", () -> "*"); - - registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); - registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); - registry.add("feast.core-authentication.options.client_id", () -> CLIENT_ID); - registry.add("feast.core-authentication.options.client_secret", () -> CLIENT_SECRET); - registry.add("feast.core-authentication.options.audience", () -> AUDIENCE); - registry.add("feast.core-authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - } -} diff --git a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java b/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java deleted file mode 100644 index f7bc12f5fc6..00000000000 --- a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; - -public class CoreSimpleAPIClient { - private CoreServiceGrpc.CoreServiceBlockingStub stub; - - public CoreSimpleAPIClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { - this.stub = stub; - } - - public void simpleApplyEntity(String projectName, EntityProto.EntitySpecV2 entitySpec) { - stub.applyEntity( - CoreServiceProto.ApplyEntityRequest.newBuilder() - .setProject(projectName) - .setSpec(entitySpec) - .build()); - } - - public EntityProto.Entity getEntity(String projectName, String name) { - return stub.getEntity( - CoreServiceProto.GetEntityRequest.newBuilder() - .setProject(projectName) - .setName(name) - .build()) - .getEntity(); - } - - public void simpleApplyFeatureTable( - String projectName, FeatureTableProto.FeatureTableSpec featureTable) { - stub.applyFeatureTable( - CoreServiceProto.ApplyFeatureTableRequest.newBuilder() - .setProject(projectName) - .setTableSpec(featureTable) - .build()); - } - - public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { - return stub.getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getTable(); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceIT.java b/serving/src/test/java/feast/serving/it/ServingServiceIT.java deleted file mode 100644 index f08ff880553..00000000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceIT.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.*; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.storage.RedisProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.*; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - }) -@Testcontainers -public class ServingServiceIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static final String timestampPrefix = "_ts"; - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - static RedisCommands syncCommands; - - static final int FEAST_SERVING_PORT = 6568; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(REDIS, REDIS_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - RedisClient redisClient = - RedisClient.create( - new RedisURI( - environment.getServiceHost("redis_1", REDIS_PORT), - environment.getServicePort("redis_1", REDIS_PORT), - java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - syncCommands = connection.sync(); - - String projectName = "default"; - // Apply Entity - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - String description = "My driver id"; - ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(entityName) - .setDescription(description) - .setValueType(entityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, entitySpec); - - // Apply FeatureTable - String featureTableName = "rides"; - ImmutableList entities = ImmutableList.of(entityName); - - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - ServingAPIProto.FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - // Event Timestamp - String eventTimestampKey = timestampPrefix + ":" + featureTableName; - Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); - - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Serialize Redis Key with Entity i.e - RedisProto.RedisKeyV2 redisKey = - RedisProto.RedisKeyV2.newBuilder() - .setProject(projectName) - .addEntityNames(entityName) - .addEntityValues(entityValue) - .build(); - - ImmutableMap featureReferenceValueMap = - ImmutableMap.of( - feature1Reference, - DataGenerator.createInt64Value(42), - feature2Reference, - DataGenerator.createDoubleValue(42.2), - feature3Reference, - DataGenerator.createEmptyValue(), - feature4Reference, - DataGenerator.createDoubleValue(42.2)); - - // Insert timestamp into Redis and isTimestampMap only once - syncCommands.hset( - redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); - featureReferenceValueMap.forEach( - (featureReference, featureValue) -> { - // Murmur hash Redis Feature Field i.e murmur() - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - byte[] featureReferenceBytes = - Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - // Insert features into Redis - syncCommands.hset( - redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); - }); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldRegisterAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ImmutableList featureReferences = - ImmutableList.of(feature1Reference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(feature1Reference), - DataGenerator.createInt64Value(42)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(feature1Reference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldRegisterAndGetOnlineFeaturesWithNotFound() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - ServingAPIProto.FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(42), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldGetOnlineFeaturesOutsideMaxAge() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 7400); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForDiffType() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnNotFoundForUpdatedType() { - String projectName = "default"; - String entityName = "driver_id"; - String featureTableName = "rides"; - - ImmutableList entities = ImmutableList.of(entityName); - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.STRING, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Sleep is necessary to ensure caching (every 1s) of updated FeatureTable is done - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - } - - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java deleted file mode 100644 index 8f2440d2475..00000000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import com.google.common.collect.ImmutableMap; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false" - }) -@Testcontainers -public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { - - CoreSimpleAPIClient coreClient; - FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; - static final Map options = new HashMap<>(); - - static final int FEAST_SERVING_PORT = 6566; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @BeforeEach - public void initState() { - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - EntityProto.EntitySpecV2 entitySpec = - DataGenerator.createEntitySpecV2( - ENTITY_ID, - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); - - expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - Arrays.asList(ENTITY_ID), - new HashMap<>() { - { - put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldAllowUnauthenticatedGetOnlineFeatures() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); - - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAuthenticated() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java deleted file mode 100644 index 64fe44b2dce..00000000000 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import feast.common.it.DataGenerator; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import sh.ory.keto.ApiException; - -@ActiveProfiles("it") -@SpringBootTest( - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true" - }) -@Testcontainers -public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { - - static final Map adminCredentials = new HashMap<>(); - static final Map memberCredentials = new HashMap<>(); - static final String PROJECT_MEMBER_CLIENT_ID = "client_id_1"; - static final String NOT_PROJECT_MEMBER_CLIENT_ID = "client_id_2"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - static String subjectClaim = "sub"; - static CoreSimpleAPIClient coreClient; - static final int FEAST_SERVING_PORT = 6766; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml"), - new File("src/test/resources/docker-compose/docker-compose-it-keto.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - AuthTestUtils.seedKeto(ketoExternalUrl, PROJECT_NAME, PROJECT_MEMBER_CLIENT_ID, CLIENT_ID); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, NOT_PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - // set up options for call credentials - adminCredentials.put("oauth_url", TOKEN_URL); - adminCredentials.put(CLIENT_ID, CLIENT_ID); - adminCredentials.put(CLIENT_SECRET, CLIENT_SECRET); - adminCredentials.put("jwkEndpointURI", JWK_URI); - adminCredentials.put("audience", AUDIENCE); - adminCredentials.put("grant_type", GRANT_TYPE); - - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); - coreClient.simpleApplyEntity( - PROJECT_NAME, - DataGenerator.createEntitySpecV2( - ENTITY_ID, "", ValueProto.ValueType.Enum.STRING, Collections.emptyMap())); - coreClient.simpleApplyFeatureTable( - PROJECT_NAME, - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - ImmutableList.of(ENTITY_ID), - ImmutableMap.of(FEATURE_NAME, ValueProto.ValueType.Enum.STRING), - 0, - Collections.emptyMap())); - } - - @Test - public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAdmin() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfProjectMember() { - Map memberCredsOptions = new HashMap<>(); - memberCredsOptions.putAll(adminCredentials); - memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void cantGetOnlineFeaturesIfNotProjectMember() { - Map notMemberCredsOptions = new HashMap<>(); - notMemberCredsOptions.putAll(adminCredentials); - notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", - PROJECT_NAME, NOT_PROJECT_MEMBER_CLIENT_ID); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/serving/src/test/java/feast/serving/it/TestUtils.java b/serving/src/test/java/feast/serving/it/TestUtils.java deleted file mode 100644 index 6772dade9c1..00000000000 --- a/serving/src/test/java/feast/serving/it/TestUtils.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.common.collect.ImmutableMap; -import feast.common.auth.credentials.OAuthCredentials; -import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.EntityProto.Entity; -import feast.proto.core.EntityProto.EntitySpecV2; -import feast.proto.core.FeatureTableProto.FeatureTable; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import java.util.*; - -public class TestUtils { - - public static ServingServiceGrpc.ServingServiceBlockingStub getServingServiceStub( - boolean isSecure, int feastServingPort, Map options) { - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feastServingPort).usePlaintext().build(); - - if (isSecure) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - return ServingServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - } else { - return ServingServiceGrpc.newBlockingStub(secureChannel); - } - } - - public static CoreSimpleAPIClient getApiClientForCore(int feastCorePort) { - Channel channel = - ManagedChannelBuilder.forAddress("localhost", feastCorePort).usePlaintext().build(); - - CoreServiceBlockingStub coreService = CoreServiceGrpc.newBlockingStub(channel); - - return new CoreSimpleAPIClient(coreService); - } - - public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( - String projectName, - List featureReferences, - List entityRows) { - return GetOnlineFeaturesRequestV2.newBuilder() - .setProject(projectName) - .addAllFeatures(featureReferences) - .addAllEntityRows(entityRows) - .build(); - } - - public static void applyFeatureTable( - CoreSimpleAPIClient secureApiClient, - String projectName, - String featureTableName, - List entities, - ImmutableMap features, - int maxAgeSecs) { - FeatureTableSpec expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - featureTableName, - entities, - features, - maxAgeSecs, - Map.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) - .build(); - secureApiClient.simpleApplyFeatureTable(projectName, expectedFeatureTableSpec); - FeatureTable actualFeatureTable = - secureApiClient.simpleGetFeatureTable(projectName, featureTableName); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - } - - public static void applyEntity( - CoreSimpleAPIClient coreApiClient, String projectName, EntitySpecV2 entitySpec) { - coreApiClient.simpleApplyEntity(projectName, entitySpec); - String entityName = entitySpec.getName(); - Entity actualEntity = coreApiClient.getEntity(projectName, entityName); - assertEquals(entitySpec.getName(), actualEntity.getSpec().getName()); - } -} diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java deleted file mode 100644 index 57932d49297..00000000000 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.FeatureTableProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto.Store; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.types.ValueProto; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreSpecService; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mock; - -public class CachedSpecServiceTest { - - private Store store; - - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - @Mock CoreSpecService coreService; - - private CachedSpecService cachedSpecService; - - private ImmutableList featureTableEntities; - private ImmutableMap featureTable1Features; - private ImmutableMap featureTable2Features; - private FeatureTableSpec featureTable1Spec; - private FeatureTableSpec featureTable2Spec; - - @Before - public void setUp() { - initMocks(this); - - this.store = Store.newBuilder().build(); - - this.setupProject("default"); - this.featureTableEntities = ImmutableList.of("entity1"); - this.featureTable1Features = - ImmutableMap.of( - "trip_cost1", ValueProto.ValueType.Enum.INT64, - "trip_distance1", ValueProto.ValueType.Enum.DOUBLE, - "trip_empty1", ValueProto.ValueType.Enum.DOUBLE); - this.featureTable2Features = - ImmutableMap.of( - "trip_cost2", ValueProto.ValueType.Enum.INT64, - "trip_distance2", ValueProto.ValueType.Enum.DOUBLE, - "trip_empty2", ValueProto.ValueType.Enum.DOUBLE); - this.featureTable1Spec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - this.featureTableEntities, - featureTable1Features, - 7200, - ImmutableMap.of()); - this.featureTable2Spec = - DataGenerator.createFeatureTableSpec( - "featuretable2", - this.featureTableEntities, - featureTable2Features, - 7200, - ImmutableMap.of()); - - this.setupFeatureTableAndProject("default"); - - when(this.coreService.registerStore(store)).thenReturn(store); - cachedSpecService = new CachedSpecService(this.coreService, this.store); - } - - private void setupProject(String project) { - when(coreService.listProjects(ListProjectsRequest.newBuilder().build())) - .thenReturn(ListProjectsResponse.newBuilder().addProjects(project).build()); - } - - private void setupFeatureTableAndProject(String project) { - FeatureTableProto.FeatureTable featureTable1 = - FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable1Spec).build(); - FeatureTableProto.FeatureTable featureTable2 = - FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable2Spec).build(); - - when(coreService.listFeatureTables( - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject(project).build()) - .build())) - .thenReturn( - ListFeatureTablesResponse.newBuilder() - .addTables(featureTable1) - .addTables(featureTable2) - .build()); - } - - @Test - public void shouldRegisterStoreWithCore() { - verify(coreService, times(1)).registerStore(cachedSpecService.getStore()); - } - - @Test - public void shouldPopulateAndReturnStore() { - cachedSpecService.populateCache(); - Store actual = cachedSpecService.getStore(); - assertThat(actual, equalTo(store)); - } - - @Test - public void shouldPopulateAndReturnDifferentFeatureTables() { - // test that CachedSpecService can retrieve fully qualified feature references. - cachedSpecService.populateCache(); - FeatureReferenceV2 featureReference1 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable1") - .setName("trip_cost1") - .build(); - FeatureReferenceV2 featureReference2 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable1") - .setName("trip_distance1") - .build(); - FeatureReferenceV2 featureReference3 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable2") - .setName("trip_empty2") - .build(); - - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference1), - equalTo(this.featureTable1Spec)); - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference2), - equalTo(this.featureTable1Spec)); - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference3), - equalTo(this.featureTable2Spec)); - } -} diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java deleted file mode 100644 index 539ed398e16..00000000000 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static feast.common.it.DataGenerator.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.ValueProto; -import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.Feature; -import feast.storage.connectors.redis.retriever.OnlineRetriever; -import io.opentracing.Tracer; -import io.opentracing.Tracer.SpanBuilder; -import java.util.ArrayList; -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.Mockito; - -public class OnlineServingServiceTest { - - @Mock CachedSpecService specService; - @Mock Tracer tracer; - @Mock OnlineRetriever retrieverV2; - - private OnlineServingServiceV2 onlineServingServiceV2; - - List mockedFeatureRows; - List featureSpecs; - - @Before - public void setUp() { - initMocks(this); - onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); - - mockedFeatureRows = new ArrayList<>(); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("1")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("2")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("3")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .setFeatureValue(createStrValue("4")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_3") - .build()) - .setFeatureValue(createStrValue("5")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) - .build()); - mockedFeatureRows.add( - Feature.builder() - .setFeatureReference( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .setFeatureValue(createStrValue("6")) - .setEventTimestamp(Timestamp.newBuilder().setSeconds(50).build()) - .build()); - - featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_1") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()); - featureSpecs.add( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_2") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()); - } - - @Test - public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { - String projectName = "default"; - ServingAPIProto.FeatureReferenceV2 featureReference1 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build(); - ServingAPIProto.FeatureReferenceV2 featureReference2 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build(); - List featureReferences = - List.of(featureReference1, featureReference2); - GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); - - List entityKeyList1 = new ArrayList<>(); - List entityKeyList2 = new ArrayList<>(); - entityKeyList1.add(mockedFeatureRows.get(0)); - entityKeyList1.add(mockedFeatureRows.get(1)); - entityKeyList2.add(mockedFeatureRows.get(2)); - entityKeyList2.add(mockedFeatureRows.get(3)); - - List> featureRows = List.of(entityKeyList1, entityKeyList2); - - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) - .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) - .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference())) - .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference())) - .thenReturn(featureSpecs.get(1)); - - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createStrValue("1")) - .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_2", createStrValue("2")) - .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createStrValue("3")) - .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_2", createStrValue("4")) - .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) - .build()) - .build(); - GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { - String projectName = "default"; - ServingAPIProto.FeatureReferenceV2 featureReference1 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build(); - ServingAPIProto.FeatureReferenceV2 featureReference2 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build(); - List featureReferences = - List.of(featureReference1, featureReference2); - GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); - - List entityKeyList1 = new ArrayList<>(); - List entityKeyList2 = new ArrayList<>(); - entityKeyList1.add(mockedFeatureRows.get(0)); - entityKeyList1.add(mockedFeatureRows.get(1)); - entityKeyList2.add(mockedFeatureRows.get(4)); - - List> featureRows = List.of(entityKeyList1, entityKeyList2); - - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) - .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) - .thenReturn(featureSpecs.get(1)); - - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createStrValue("1")) - .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_2", createStrValue("2")) - .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createEmptyValue()) - .putStatuses("featuretable_1:feature_1", FieldStatus.NOT_FOUND) - .putFields("featuretable_1:feature_2", createEmptyValue()) - .putStatuses("featuretable_1:feature_2", FieldStatus.NOT_FOUND) - .build()) - .build(); - GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { - String projectName = "default"; - ServingAPIProto.FeatureReferenceV2 featureReference1 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build(); - ServingAPIProto.FeatureReferenceV2 featureReference2 = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build(); - List featureReferences = - List.of(featureReference1, featureReference2); - GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); - - List entityKeyList1 = new ArrayList<>(); - List entityKeyList2 = new ArrayList<>(); - entityKeyList1.add(mockedFeatureRows.get(5)); - entityKeyList1.add(mockedFeatureRows.get(1)); - entityKeyList2.add(mockedFeatureRows.get(5)); - entityKeyList2.add(mockedFeatureRows.get(1)); - - List> featureRows = List.of(entityKeyList1, entityKeyList2); - - when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())) - .thenReturn( - FeatureTableSpec.newBuilder() - .setName("featuretable_1") - .addEntities("entity1") - .addEntities("entity2") - .addFeatures( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_1") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .addFeatures( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_2") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .setMaxAge(Duration.newBuilder().setSeconds(1)) - .build()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) - .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference())) - .thenReturn(featureSpecs.get(0)); - - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createEmptyValue()) - .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) - .putFields("featuretable_1:feature_2", createStrValue("2")) - .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", createInt64Value(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", createStrValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createEmptyValue()) - .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) - .putFields("featuretable_1:feature_2", createStrValue("2")) - .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) - .build()) - .build(); - GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } - - private FeatureTableSpec getFeatureTableSpec() { - return FeatureTableSpec.newBuilder() - .setName("featuretable_1") - .addEntities("entity1") - .addEntities("entity2") - .addFeatures( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_1") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .addFeatures( - FeatureProto.FeatureSpecV2.newBuilder() - .setName("feature_2") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .setMaxAge(Duration.newBuilder().setSeconds(120)) - .build(); - } - - private GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2( - String projectName, List featureReferences) { - return GetOnlineFeaturesRequestV2.newBuilder() - .setProject(projectName) - .addAllFeatures(featureReferences) - .addEntityRows( - GetOnlineFeaturesRequestV2.EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", createInt64Value(1)) - .putFields("entity2", createStrValue("a"))) - .addEntityRows( - GetOnlineFeaturesRequestV2.EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", createInt64Value(2)) - .putFields("entity2", createStrValue("b"))) - .addFeatures( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_1") - .build()) - .addFeatures( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable_1") - .setName("feature_2") - .build()) - .build(); - } -} diff --git a/serving/src/test/java/feast/serving/util/RequestHelperTest.java b/serving/src/test/java/feast/serving/util/RequestHelperTest.java deleted file mode 100644 index 140d46cd569..00000000000 --- a/serving/src/test/java/feast/serving/util/RequestHelperTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.util; - -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import org.junit.Test; - -public class RequestHelperTest { - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfEntityRowEmpty() { - FeatureReferenceV2 featureReference = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretablename") - .setName("featurename") - .build(); - GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 = - GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build(); - RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfFeatureReferenceTableEmpty() { - FeatureReferenceV2 featureReference = - FeatureReferenceV2.newBuilder().setName("featurename").build(); - GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 = - GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build(); - RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfFeatureReferenceNameEmpty() { - FeatureReferenceV2 featureReference = - FeatureReferenceV2.newBuilder().setFeatureTable("featuretablename").build(); - GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2 = - GetOnlineFeaturesRequestV2.newBuilder().addFeatures(featureReference).build(); - RequestHelper.validateOnlineRequest(getOnlineFeaturesRequestV2); - } -} diff --git a/serving/src/test/resources/application-it.properties b/serving/src/test/resources/application-it.properties deleted file mode 100644 index 000e512a680..00000000000 --- a/serving/src/test/resources/application-it.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -feast.core-authentication.enabled=false -feast.security.authentication.enabled=false -feast.security.authorization.enabled=false \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/core/application-it.yml b/serving/src/test/resources/docker-compose/core/application-it.yml deleted file mode 100644 index 1298ff99055..00000000000 --- a/serving/src/test/resources/docker-compose/core/application-it.yml +++ /dev/null @@ -1,13 +0,0 @@ -feast: - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" - - security: - authentication: - enabled: true - provider: jwt - options: - jwkEndpointURI: http://hydra:4444/.well-known/jwks.json \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml b/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml deleted file mode 100644 index 1c20610cc73..00000000000 --- a/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: '3' - -services: - hydra-migrate: - image: oryd/hydra:v1.6.0 - environment: - - DSN=postgres://hydra:secret@postgresd:5432/hydra?sslmode=disable&max_conns=20&max_idle_conns=4 - command: - migrate sql -e --yes - restart: on-failure - - hydra: - depends_on: - - hydra-migrate - environment: - - DSN=postgres://hydra:secret@postgresd:5432/hydra?sslmode=disable&max_conns=20&max_idle_conns=4 - - postgresd: - image: postgres:9.6 - ports: - - "54320:5432" - environment: - - POSTGRES_USER=hydra - - POSTGRES_PASSWORD=secret - - POSTGRES_DB=hydra - - hydra: - image: oryd/hydra:v1.6.0 - ports: - - "4444:4444" # Public port - - "4445:4445" # Admin port - #- "5555:5555" # Port for hydra token user - command: - serve all --dangerous-force-http - environment: - - URLS_SELF_ISSUER=http://hydra:4444 - - URLS_CONSENT=http://hydra:3000/consent - - URLS_LOGIN=http://hydra:3000/login - - URLS_LOGOUT=http://hydra:3000/logout - - DSN=memory - - SECRETS_SYSTEM=youReallyNeedToChangeThis - - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public,pairwise - - OIDC_SUBJECT_IDENTIFIERS_PAIRWISE_SALT=youReallyNeedToChangeThis - - OAUTH2_ACCESS_TOKEN_STRATEGY=jwt - - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public - restart: unless-stopped - - consent: - environment: - - HYDRA_ADMIN_URL=http://hydra:4445 - image: oryd/hydra-login-consent-node:v1.5.2 - ports: - - "3000:3000" - restart: unless-stopped diff --git a/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml b/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml deleted file mode 100644 index 8ebf7f225e0..00000000000 --- a/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3' -services: - keto: - depends_on: - - ketodb - - migrations - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@ketodb:5432/keto?sslmode=disable - command: - - serve - ports: - - 4466 - - ketodb: - image: bitnami/postgresql:9.6 - environment: - - POSTGRESQL_USERNAME=keto - - POSTGRESQL_PASSWORD=keto - - POSTGRESQL_DATABASE=keto - ports: - - "54340:5432" - - migrations: - depends_on: - - ketodb - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@ketodb:5432/keto?sslmode=disable - command: - - migrate - - sql - - -e - - adaptor: - depends_on: - - keto - image: gcr.io/kf-feast/feast-keto-auth-server:latest - environment: - SERVER_PORT: 8080 - KETO_URL: http://keto:4466 - ports: - - 8080 - restart: on-failure \ No newline at end of file diff --git a/serving/src/test/resources/docker-compose/docker-compose-it.yml b/serving/src/test/resources/docker-compose/docker-compose-it.yml deleted file mode 100644 index fb7fb1f6adc..00000000000 --- a/serving/src/test/resources/docker-compose/docker-compose-it.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: '3' - -services: - core: - image: gcr.io/kf-feast/feast-core:develop - volumes: - - ./core/application-it.yml:/etc/feast/application.yml - environment: - DB_HOST: db - restart: on-failure - depends_on: - - db - - kafka - ports: - - 6565:6565 - command: - - java - - -jar - - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - kafka: - image: confluentinc/cp-kafka:5.2.1 - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092,OUTSIDE://localhost:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - ports: - - "9092:9092" - - "9094:9094" - - depends_on: - - zookeeper - - zookeeper: - image: confluentinc/cp-zookeeper:5.2.1 - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - - db: - image: postgres:12-alpine - environment: - POSTGRES_PASSWORD: password - ports: - - "5432:5432" - - redis: - image: redis:5-alpine - ports: - - "6379:6379" \ No newline at end of file diff --git a/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8e..00000000000 --- a/serving/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file diff --git a/storage/api/pom.xml b/storage/api/pom.xml deleted file mode 100644 index cc2f84ecb17..00000000000 --- a/storage/api/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - dev.feast - feast-parent - ${revision} - ../.. - - - 4.0.0 - feast-storage-api - - Feast Storage API - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - - javax.annotation - - - - - - - - - - dev.feast - datatypes-java - ${project.version} - - - - dev.feast - feast-common - ${project.version} - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - - com.google.auto.value - auto-value - 1.6.6 - provided - - - - org.apache.commons - commons-lang3 - 3.9 - - - - junit - junit - 4.12 - test - - - - diff --git a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java b/storage/api/src/main/java/feast/storage/api/retriever/Feature.java deleted file mode 100644 index c6cee0883e1..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/Feature.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import com.google.auto.value.AutoValue; -import com.google.protobuf.Timestamp; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.types.ValueProto.Value; - -@AutoValue -public abstract class Feature { - - public abstract FeatureReferenceV2 getFeatureReference(); - - public abstract Value getFeatureValue(); - - public abstract Timestamp getEventTimestamp(); - - public static Builder builder() { - return new AutoValue_Feature.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setFeatureReference(FeatureReferenceV2 featureReference); - - public abstract Builder setFeatureValue(Value featureValue); - - public abstract Builder setEventTimestamp(Timestamp eventTimestamp); - - public abstract Feature build(); - } -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java b/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java deleted file mode 100644 index 6188a270c40..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableSet; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@AutoValue -public abstract class FeatureTableRequest { - public abstract FeatureTableSpec getSpec(); - - public abstract ImmutableSet getFeatureReferences(); - - public static Builder newBuilder() { - return new AutoValue_FeatureTableRequest.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSpec(FeatureTableSpec spec); - - abstract ImmutableSet.Builder featureReferencesBuilder(); - - public Builder addAllFeatureReferences(List featureReferenceList) { - featureReferencesBuilder().addAll(featureReferenceList); - return this; - } - - public Builder addFeatureReference(FeatureReferenceV2 featureReference) { - featureReferencesBuilder().add(featureReference); - return this; - } - - public abstract FeatureTableRequest build(); - } - - public Map getFeatureRefsByName() { - return getFeatureReferences().stream() - .collect( - Collectors.toMap(FeatureReferenceV2::getName, featureReference -> featureReference)); - } -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java deleted file mode 100644 index 9be66a7b1fb..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import feast.proto.serving.ServingAPIProto; -import java.util.List; - -public interface OnlineRetrieverV2 { - /** - * Get online features for the given entity rows using data retrieved from the Feature references - * specified in FeatureTable request. - * - *

Each {@link Feature} optional in the returned list then corresponds to an {@link - * ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} provided by the user. If feature for a - * given entity row is not found, will return an empty optional instead. The no. of {@link - * Feature} returned should match the no. of given {@link - * ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s - * - * @param project name of project to request features from. - * @param entityRows list of entity rows to request features for. - * @param featureReferences specifies the FeatureTable to retrieve data from - * @return list of {@link Feature}s corresponding to data retrieved for each entity row from - * FeatureTable specified in FeatureTable request. - */ - List> getOnlineFeatures( - String project, - List entityRows, - List featureReferences); -} diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml deleted file mode 100644 index 4969364b623..00000000000 --- a/storage/connectors/pom.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - dev.feast - feast-parent - ${revision} - ../.. - - - 4.0.0 - feast-storage-connectors - pom - - Feast Storage Connectors - - - redis - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - - javax.annotation - - - - - - org.jacoco - jacoco-maven-plugin - - - - - - - dev.feast - datatypes-java - ${project.version} - - - - dev.feast - feast-common - ${project.version} - - - - dev.feast - feast-storage-api - ${project.version} - - - - diff --git a/storage/connectors/redis/pom.xml b/storage/connectors/redis/pom.xml deleted file mode 100644 index bbda8dab27f..00000000000 --- a/storage/connectors/redis/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - dev.feast - feast-storage-connectors - ${revision} - - - 4.0.0 - feast-storage-connector-redis - - Feast Storage Connector for Redis - - - - io.lettuce - lettuce-core - - - - org.apache.commons - commons-lang3 - 3.9 - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - - com.google.auto.value - auto-value - 1.6.6 - provided - - - - com.google.guava - guava - - - - org.mockito - mockito-core - ${mockito.version} - test - - - - - com.github.kstyrc - embedded-redis - test - - - - org.hamcrest - hamcrest-core - test - - - - org.hamcrest - hamcrest-library - test - - - - net.ishiis.redis - redis-unit - 1.0.3 - test - - - - - junit - junit - 4.12 - test - - - org.slf4j - slf4j-simple - 1.7.30 - test - - - diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java deleted file mode 100644 index 44f74d3f56b..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.common; - -import com.google.common.hash.Hashing; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import feast.proto.serving.ServingAPIProto; -import feast.proto.types.ValueProto; -import feast.storage.api.retriever.Feature; -import io.lettuce.core.KeyValue; -import java.nio.charset.StandardCharsets; -import java.util.*; - -public class RedisHashDecoder { - - /** - * Converts all retrieved Redis Hash values based on EntityRows into {@link Feature} - * - * @param redisHashValues retrieved Redis Hash values based on EntityRows - * @param byteToFeatureReferenceMap map to decode bytes back to FeatureReference - * @return List of {@link Feature} - * @throws InvalidProtocolBufferException - */ - public static List retrieveFeature( - List> redisHashValues, - Map byteToFeatureReferenceMap, - String timestampPrefix) - throws InvalidProtocolBufferException { - List allFeatures = new ArrayList<>(); - Map allFeaturesBuilderMap = - new HashMap<>(); - Map featureTableTimestampMap = new HashMap<>(); - - for (KeyValue entity : redisHashValues) { - if (entity.hasValue()) { - byte[] redisValueK = entity.getKey(); - byte[] redisValueV = entity.getValue(); - - // Decode data from Redis into Feature object fields - if (new String(redisValueK).startsWith(timestampPrefix)) { - Timestamp eventTimestamp = Timestamp.parseFrom(redisValueV); - featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); - } else { - ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK.toString()); - ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); - - Feature.Builder featureBuilder = - Feature.builder().setFeatureReference(featureReference).setFeatureValue(featureValue); - allFeaturesBuilderMap.put(featureReference, featureBuilder); - } - } - } - - // Add timestamp to features - for (Map.Entry entry : - allFeaturesBuilderMap.entrySet()) { - String timestampRedisHashKeyStr = timestampPrefix + ":" + entry.getKey().getFeatureTable(); - Timestamp curFeatureTimestamp = featureTableTimestampMap.get(timestampRedisHashKeyStr); - - Feature curFeature = entry.getValue().setEventTimestamp(curFeatureTimestamp).build(); - allFeatures.add(curFeature); - } - - return allFeatures; - } - - public static byte[] getTimestampRedisHashKeyBytes( - ServingAPIProto.FeatureReferenceV2 featureReference, String timestampPrefix) { - String timestampRedisHashKeyStr = timestampPrefix + ":" + featureReference.getFeatureTable(); - return timestampRedisHashKeyStr.getBytes(); - } - - public static byte[] getFeatureReferenceRedisHashKeyBytes( - ServingAPIProto.FeatureReferenceV2 featureReference) { - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - return Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java deleted file mode 100644 index 797dd522151..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.common; - -import feast.proto.serving.ServingAPIProto; -import feast.proto.storage.RedisProto; -import feast.proto.types.ValueProto; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class RedisKeyGenerator { - - public static List buildRedisKeys( - String project, List entityRows) { - List redisKeys = - entityRows.stream() - .map(entityRow -> makeRedisKey(project, entityRow)) - .collect(Collectors.toList()); - - return redisKeys; - } - - /** - * Create {@link RedisProto.RedisKeyV2} - * - * @param project Project where request for features was called from - * @param entityRow {@link ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} - * @return {@link RedisProto.RedisKeyV2} - */ - private static RedisProto.RedisKeyV2 makeRedisKey( - String project, ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow) { - RedisProto.RedisKeyV2.Builder builder = RedisProto.RedisKeyV2.newBuilder().setProject(project); - Map fieldsMap = entityRow.getFieldsMap(); - List entityNames = new ArrayList<>(new HashSet<>(fieldsMap.keySet())); - - // Sort entity names by alphabetical order - entityNames.sort(String::compareTo); - - for (String entityName : entityNames) { - builder.addEntityNames(entityName); - builder.addEntityValues(fieldsMap.get(entityName)); - } - return builder.build(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java deleted file mode 100644 index 79d00240a00..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import com.google.common.collect.Lists; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.serving.ServingAPIProto; -import feast.proto.storage.RedisProto; -import feast.storage.api.retriever.Feature; -import feast.storage.api.retriever.OnlineRetrieverV2; -import feast.storage.connectors.redis.common.RedisHashDecoder; -import feast.storage.connectors.redis.common.RedisKeyGenerator; -import io.grpc.Status; -import io.lettuce.core.KeyValue; -import io.lettuce.core.RedisFuture; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -public class OnlineRetriever implements OnlineRetrieverV2 { - - private static final String timestampPrefix = "_ts"; - private RedisClientAdapter redisClientAdapter; - - public OnlineRetriever(RedisClientAdapter redisClientAdapter) { - this.redisClientAdapter = redisClientAdapter; - } - - @Override - public List> getOnlineFeatures( - String project, - List entityRows, - List featureReferences) { - - List redisKeys = RedisKeyGenerator.buildRedisKeys(project, entityRows); - List> features = getFeaturesFromRedis(redisKeys, featureReferences); - - return features; - } - - private List> getFeaturesFromRedis( - List redisKeys, - List featureReferences) { - List> features = new ArrayList<>(); - // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); - - // Serialize using proto - List binaryRedisKeys = - redisKeys.stream().map(redisKey -> redisKey.toByteArray()).collect(Collectors.toList()); - - List featureReferenceWithTsByteList = new ArrayList<>(); - featureReferences.stream() - .forEach( - featureReference -> { - - // eg. murmur() - byte[] featureReferenceBytes = - RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); - featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); - - // eg. <_ts:featuretable_name> - byte[] featureTableTsBytes = - RedisHashDecoder.getTimestampRedisHashKeyBytes(featureReference, timestampPrefix); - featureReferenceWithTsByteList.add(featureTableTsBytes); - }); - - // Perform a series of independent calls - List>>> futures = Lists.newArrayList(); - for (byte[] binaryRedisKey : binaryRedisKeys) { - byte[][] featureReferenceWithTsByteArrays = - featureReferenceWithTsByteList.toArray(new byte[0][]); - // Access redis keys and extract features - futures.add(redisClientAdapter.hmget(binaryRedisKey, featureReferenceWithTsByteArrays)); - } - - // Write all commands to the transport layer - redisClientAdapter.flushCommands(); - - futures.forEach( - future -> { - try { - List> redisValuesList = future.get(); - List curRedisKeyFeatures = - RedisHashDecoder.retrieveFeature( - redisValuesList, byteToFeatureReferenceMap, timestampPrefix); - features.add(curRedisKeyFeatures); - } catch (InterruptedException | ExecutionException | InvalidProtocolBufferException e) { - throw Status.UNKNOWN - .withDescription("Unexpected error when pulling data from from Redis.") - .withCause(e) - .asRuntimeException(); - } - }); - return features; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java deleted file mode 100644 index 5a7f4b78736..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import feast.proto.core.StoreProto; -import io.lettuce.core.KeyValue; -import io.lettuce.core.RedisFuture; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.List; - -public class RedisClient implements RedisClientAdapter { - - private final RedisAsyncCommands asyncCommands; - - @Override - public RedisFuture>> hmget(byte[] key, byte[]... fields) { - return asyncCommands.hmget(key, fields); - } - - @Override - public void flushCommands() { - asyncCommands.flushCommands(); - } - - private RedisClient(StatefulRedisConnection connection) { - this.asyncCommands = connection.async(); - - // Disable auto-flushing - this.asyncCommands.setAutoFlushCommands(false); - } - - public static RedisClientAdapter create(StoreProto.Store.RedisConfig config) { - - RedisURI uri = RedisURI.create(config.getHost(), config.getPort()); - - if (config.getSsl()) { - uri.setSsl(true); - } - StatefulRedisConnection connection = - io.lettuce.core.RedisClient.create(uri).connect(new ByteArrayCodec()); - - return new RedisClient(connection); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClientAdapter.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClientAdapter.java deleted file mode 100644 index 65e730ae930..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClientAdapter.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import io.lettuce.core.*; -import java.util.List; - -public interface RedisClientAdapter { - RedisFuture>> hmget(byte[] key, byte[]... fields); - - void flushCommands(); -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java deleted file mode 100644 index aeb8220b0cb..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import com.google.common.collect.ImmutableMap; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializerV2; -import feast.storage.connectors.redis.serializer.RedisKeySerializerV2; -import io.lettuce.core.KeyValue; -import io.lettuce.core.ReadFrom; -import io.lettuce.core.RedisFuture; -import io.lettuce.core.RedisURI; -import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nullable; - -public class RedisClusterClient implements RedisClientAdapter { - - private final RedisAdvancedClusterAsyncCommands asyncCommands; - private final RedisKeySerializerV2 serializer; - @Nullable private final RedisKeySerializerV2 fallbackSerializer; - - private static final Map PROTO_TO_LETTUCE_TYPES = - ImmutableMap.of( - RedisClusterConfig.ReadFrom.MASTER, ReadFrom.MASTER, - RedisClusterConfig.ReadFrom.MASTER_PREFERRED, ReadFrom.MASTER_PREFERRED, - RedisClusterConfig.ReadFrom.REPLICA, ReadFrom.REPLICA, - RedisClusterConfig.ReadFrom.REPLICA_PREFERRED, ReadFrom.REPLICA_PREFERRED); - - @Override - public RedisFuture>> hmget(byte[] key, byte[]... fields) { - return asyncCommands.hmget(key, fields); - } - - @Override - public void flushCommands() { - asyncCommands.flushCommands(); - } - - static class Builder { - private final StatefulRedisClusterConnection connection; - private final RedisKeySerializerV2 serializer; - @Nullable private RedisKeySerializerV2 fallbackSerializer; - - Builder( - StatefulRedisClusterConnection connection, - RedisKeySerializerV2 serializer) { - this.connection = connection; - this.serializer = serializer; - } - - Builder withFallbackSerializer(RedisKeySerializerV2 fallbackSerializer) { - this.fallbackSerializer = fallbackSerializer; - return this; - } - - RedisClusterClient build() { - return new RedisClusterClient(this); - } - } - - private RedisClusterClient(Builder builder) { - this.asyncCommands = builder.connection.async(); - this.serializer = builder.serializer; - this.fallbackSerializer = builder.fallbackSerializer; - - // allows reading from replicas - this.asyncCommands.readOnly(); - - // Disable auto-flushing - this.asyncCommands.setAutoFlushCommands(false); - } - - public static RedisClientAdapter create(StoreProto.Store.RedisClusterConfig config) { - List redisURIList = - Arrays.stream(config.getConnectionString().split(",")) - .map( - hostPort -> { - String[] hostPortSplit = hostPort.trim().split(":"); - return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); - }) - .collect(Collectors.toList()); - StatefulRedisClusterConnection connection = - io.lettuce.core.cluster.RedisClusterClient.create(redisURIList) - .connect(new ByteArrayCodec()); - - connection.setReadFrom(PROTO_TO_LETTUCE_TYPES.get(config.getReadFrom())); - - RedisKeySerializerV2 serializer = new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); - - Builder builder = new Builder(connection, serializer); - - if (config.getEnableFallback()) { - RedisKeySerializerV2 fallbackSerializer = - new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); - builder = builder.withFallbackSerializer(fallbackSerializer); - } - - return builder.build(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java deleted file mode 100644 index 1c869b4fcca..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerV2.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKeyV2; - -public class RedisKeyPrefixSerializerV2 implements RedisKeySerializerV2 { - - private final byte[] prefixBytes; - - public RedisKeyPrefixSerializerV2(String prefix) { - this.prefixBytes = prefix.getBytes(); - } - - public byte[] serialize(RedisKeyV2 redisKey) { - byte[] key = redisKey.toByteArray(); - - if (prefixBytes.length == 0) { - return key; - } - - byte[] keyWithPrefix = new byte[prefixBytes.length + key.length]; - System.arraycopy(prefixBytes, 0, keyWithPrefix, 0, prefixBytes.length); - System.arraycopy(key, 0, keyWithPrefix, prefixBytes.length, key.length); - return keyWithPrefix; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java deleted file mode 100644 index 252d6d1422c..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializerV2.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKeyV2; - -public class RedisKeyProtoSerializerV2 implements RedisKeySerializerV2 { - - public byte[] serialize(RedisKeyV2 redisKey) { - return redisKey.toByteArray(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java deleted file mode 100644 index b79e1581dad..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializerV2.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKeyV2; - -public interface RedisKeySerializerV2 { - - byte[] serialize(RedisKeyV2 key); -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java deleted file mode 100644 index e663cf81ee4..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import static org.junit.Assert.*; - -import feast.proto.storage.RedisProto.RedisKeyV2; -import feast.proto.types.ValueProto; -import org.junit.Test; - -public class RedisKeyPrefixSerializerTest { - - private RedisKeyV2 key = - RedisKeyV2.newBuilder() - .addEntityNames("entity1") - .addEntityValues(ValueProto.Value.newBuilder().setInt64Val(1)) - .build(); - - @Test - public void shouldPrependKey() { - RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2("namespace:"); - String keyWithPrefix = new String(serializer.serialize(key)); - assertEquals(String.format("namespace:%s", new String(key.toByteArray())), keyWithPrefix); - } - - @Test - public void shouldNotPrependKeyIfEmptyString() { - RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2(""); - assertArrayEquals(key.toByteArray(), serializer.serialize(key)); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java deleted file mode 100644 index 66aba44bc20..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.test; - -import java.io.IOException; -import redis.embedded.RedisServer; - -public class TestUtil { - public static class LocalRedis { - - private static RedisServer server; - - /** - * Start local Redis for used in testing at "localhost" - * - * @param port port number - * @throws IOException if Redis failed to start - */ - public static void start(int port) throws IOException { - server = new RedisServer(port); - server.start(); - } - - public static void stop() { - if (server != null) { - server.stop(); - } - } - } -}