From a824951696c4e8f2911f2c805a3304ce31034faa Mon Sep 17 00:00:00 2001 From: pyalex Date: Mon, 17 Aug 2020 12:17:53 +0800 Subject: [PATCH 01/26] moving jc --- common-test/pom.xml | 153 ++++++++ .../main/java/feast/common}/it/BaseIT.java | 31 +- .../java/feast/common}/it/DataGenerator.java | 2 +- .../feast/common}/it/SimpleAPIClient.java | 4 +- .../main/java/feast/common}/it/SimpleIT.java | 2 +- .../main/java/feast/common/util/TestUtil.java | 40 +++ common/pom.xml | 14 +- .../common/models/FeatureSetReference.java | 14 + .../common}/util/KafkaSerialization.java | 2 +- core/pom.xml | 8 +- .../feast/core/config/FeastProperties.java | 180 ---------- .../core/config/FeatureStreamConfig.java | 137 ------- .../java/feast/core/grpc/CoreServiceImpl.java | 138 ++++---- core/src/main/resources/application.yml | 70 ---- .../feast/core/auth/CoreServiceAuthTest.java | 10 +- .../auth/CoreServiceAuthenticationIT.java | 6 +- .../core/auth/CoreServiceAuthorizationIT.java | 6 +- .../feast/core/metrics/CoreMetricsIT.java | 2 +- .../feast/core/service/SpecServiceIT.java | 6 +- .../test/java/feast/core/util/TestUtil.java | 150 -------- .../test/resources/application-it.properties | 5 - job-coordinator/pom.xml | 335 ++++++++++++++++++ .../feast/jc/JobCoordinatorApplication.java | 42 +++ .../java/feast/jc/config/FeastProperties.java | 315 ++++++++++++++++ .../feast/jc/config/FeatureStreamConfig.java | 162 +++++++++ .../feast/jc/config/JobCoordinatorConfig.java | 47 ++- .../feast/jc/dao}/InMemoryJobRepository.java | 7 +- .../java/feast/jc/dao}/JobRepository.java | 6 +- .../jc}/exception/JobExecutionException.java | 2 +- .../jc}/exception/JobMonitoringException.java | 2 +- .../jc}/model/FeatureSetDeliveryStatus.java | 2 +- .../src/main/java/feast/jc}/model/Job.java | 2 +- .../main/java/feast/jc}/model/JobStatus.java | 2 +- .../jc/runner}/ConsolidatedJobStrategy.java | 7 +- .../feast/jc/runner}/JobGroupingStrategy.java | 4 +- .../java/feast/jc/runner}/JobManager.java | 6 +- .../feast/jc/runner}/JobPerStoreStrategy.java | 7 +- .../main/java/feast/jc/runner}/Runner.java | 2 +- .../runner}/dataflow/DataflowJobManager.java | 27 +- .../jc/runner}/dataflow/DataflowJobState.java | 2 +- .../dataflow/DataflowJobStateMapper.java | 16 +- .../jc/runner}/dataflow/DataflowJobType.java | 2 +- .../dataflow/DataflowRunnerConfig.java | 7 +- .../feast/jc/runner}/direct/DirectJob.java | 2 +- .../jc/runner}/direct/DirectJobRegistry.java | 2 +- .../runner}/direct/DirectJobStateMapper.java | 4 +- .../jc/runner}/direct/DirectRunnerConfig.java | 4 +- .../direct/DirectRunnerJobManager.java | 28 +- .../option/FeatureSetJsonByteConverter.java | 2 +- .../feast/jc/runner}/option/RunnerConfig.java | 8 +- .../feast/jc/runner}/task/CreateJobTask.java | 10 +- .../java/feast/jc/runner}/task/JobTask.java | 8 +- .../java/feast/jc/runner}/task/JobTasks.java | 2 +- .../feast/jc/runner}/task/RestartJobTask.java | 8 +- .../jc/runner}/task/TerminateJobTask.java | 8 +- .../jc/runner}/task/UpdateJobStatusTask.java | 8 +- .../jc}/service/JobCoordinatorService.java | 102 +++--- .../java/feast/jc}/service/JobService.java | 29 +- .../main/java/feast/jc}/util/PackageUtil.java | 6 +- .../java/feast/jc}/util/PipelineUtil.java | 4 +- .../java/feast/jc/util/TypeConversion.java | 73 ++++ .../src/main/resources/application.yml | 133 +++++++ job-coordinator/src/main/resources/banner.txt | 21 ++ .../java/feast/jc}/model/JobStatusTest.java | 6 +- .../java/feast/jc/runner}/RunnerTest.java | 12 +- .../dataflow/DataflowJobManagerTest.java | 36 +- .../dataflow/DataflowJobStateMapperTest.java | 10 +- .../dataflow/DataflowRunnerConfigTest.java | 6 +- .../direct/DirectRunnerConfigTest.java | 6 +- .../direct/DirectRunnerJobManagerTest.java | 23 +- .../FeatureSetJsonByteConverterTest.java | 6 +- .../feast/jc/runner}/task/JobTasksTest.java | 21 +- .../feast/jc}/service/FakeJobManager.java | 10 +- .../feast/jc}/service/JobCoordinatorIT.java | 59 ++- .../service/JobCoordinatorServiceTest.java | 91 +++-- .../java/feast/jc}/service/JobServiceIT.java | 18 +- .../test/resources/application-it-core.yml | 67 ++++ .../test/resources/application-it.properties | 31 ++ .../org.mockito.plugins.MockMaker | 1 + pom.xml | 2 + 80 files changed, 1882 insertions(+), 969 deletions(-) create mode 100644 common-test/pom.xml rename {core/src/test/java/feast/core => common-test/src/main/java/feast/common}/it/BaseIT.java (85%) rename {core/src/test/java/feast/core => common-test/src/main/java/feast/common}/it/DataGenerator.java (99%) rename {core/src/test/java/feast/core => common-test/src/main/java/feast/common}/it/SimpleAPIClient.java (98%) rename {core/src/test/java/feast/core => common-test/src/main/java/feast/common}/it/SimpleIT.java (97%) create mode 100644 common-test/src/main/java/feast/common/util/TestUtil.java rename {core/src/main/java/feast/core => common/src/main/java/feast/common}/util/KafkaSerialization.java (98%) delete mode 100644 core/src/test/java/feast/core/util/TestUtil.java create mode 100644 job-coordinator/pom.xml create mode 100644 job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java create mode 100644 job-coordinator/src/main/java/feast/jc/config/FeastProperties.java create mode 100644 job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java rename core/src/main/java/feast/core/config/JobConfig.java => job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java (77%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/dao}/InMemoryJobRepository.java (97%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/dao}/JobRepository.java (94%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/exception/JobExecutionException.java (97%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/exception/JobMonitoringException.java (97%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/model/FeatureSetDeliveryStatus.java (98%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/model/Job.java (99%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/model/JobStatus.java (99%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/ConsolidatedJobStrategy.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/JobGroupingStrategy.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/JobManager.java (95%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/JobPerStoreStrategy.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/Runner.java (98%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/dataflow/DataflowJobManager.java (95%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/dataflow/DataflowJobState.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/dataflow/DataflowJobStateMapper.java (72%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/dataflow/DataflowJobType.java (95%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/dataflow/DataflowRunnerConfig.java (97%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/direct/DirectJob.java (97%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/direct/DirectJobRegistry.java (98%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/direct/DirectJobStateMapper.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/direct/DirectRunnerConfig.java (94%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/direct/DirectRunnerJobManager.java (91%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/option/FeatureSetJsonByteConverter.java (98%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/option/RunnerConfig.java (92%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/CreateJobTask.java (92%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/JobTask.java (94%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/JobTasks.java (96%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/RestartJobTask.java (92%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/TerminateJobTask.java (92%) rename {core/src/main/java/feast/core/job => job-coordinator/src/main/java/feast/jc/runner}/task/UpdateJobStatusTask.java (90%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/service/JobCoordinatorService.java (88%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/service/JobService.java (90%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/util/PackageUtil.java (96%) rename {core/src/main/java/feast/core => job-coordinator/src/main/java/feast/jc}/util/PipelineUtil.java (96%) create mode 100644 job-coordinator/src/main/java/feast/jc/util/TypeConversion.java create mode 100644 job-coordinator/src/main/resources/application.yml create mode 100644 job-coordinator/src/main/resources/banner.txt rename {core/src/test/java/feast/core => job-coordinator/src/test/java/feast/jc}/model/JobStatusTest.java (92%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/RunnerTest.java (79%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/dataflow/DataflowJobManagerTest.java (93%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/dataflow/DataflowJobStateMapperTest.java (78%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/dataflow/DataflowRunnerConfigTest.java (97%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/direct/DirectRunnerConfigTest.java (92%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/direct/DirectRunnerJobManagerTest.java (92%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/option/FeatureSetJsonByteConverterTest.java (96%) rename {core/src/test/java/feast/core/job => job-coordinator/src/test/java/feast/jc/runner}/task/JobTasksTest.java (93%) rename {core/src/test/java/feast/core => job-coordinator/src/test/java/feast/jc}/service/FakeJobManager.java (92%) rename {core/src/test/java/feast/core => job-coordinator/src/test/java/feast/jc}/service/JobCoordinatorIT.java (86%) rename {core/src/test/java/feast/core => job-coordinator/src/test/java/feast/jc}/service/JobCoordinatorServiceTest.java (82%) rename {core/src/test/java/feast/core => job-coordinator/src/test/java/feast/jc}/service/JobServiceIT.java (95%) create mode 100644 job-coordinator/src/test/resources/application-it-core.yml create mode 100644 job-coordinator/src/test/resources/application-it.properties create mode 100644 job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/common-test/pom.xml b/common-test/pom.xml new file mode 100644 index 00000000000..8f7efb8c79f --- /dev/null +++ b/common-test/pom.xml @@ -0,0 +1,153 @@ + + + + 4.0.0 + + + feast-parent + dev.feast + ${revision} + + + Feast Common Test + Feast common module with test utilities + feast-common-test + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true + + + + + + + + dev.feast + datatypes-java + ${project.version} + compile + + + dev.feast + feast-common + ${project.version} + compile + + + com.google.protobuf + protobuf-java-util + + + + org.projectlombok + lombok + ${lombok.version} + + + javax.validation + validation-api + + + com.google.auto.value + auto-value-annotations + + + com.google.code.gson + gson + + + + + org.slf4j + slf4j-api + + + + org.hamcrest + hamcrest-library + + + org.springframework.boot + spring-boot-test + 2.3.1.RELEASE + + + org.springframework + spring-test + 5.2.5.RELEASE + + + org.testcontainers + junit-jupiter + 1.14.3 + + + org.testcontainers + postgresql + 1.14.3 + + + org.testcontainers + kafka + 1.14.3 + + + org.junit.jupiter + junit-jupiter-api + 5.6.2 + + + org.springframework.kafka + spring-kafka + + + io.prometheus + simpleclient + 0.8.0 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.awaitility + awaitility + 3.0.0 + + + org.awaitility + awaitility-proxy + 3.0.0 + + + org.mockito + mockito-core + ${mockito.version} + compile + + + diff --git a/core/src/test/java/feast/core/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java similarity index 85% rename from core/src/test/java/feast/core/it/BaseIT.java rename to common-test/src/main/java/feast/common/it/BaseIT.java index e2e6f0c5a69..529dc85905c 100644 --- a/core/src/test/java/feast/core/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -14,11 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.it; +package feast.common.it; -import feast.core.config.FeastProperties; -import feast.core.util.KafkaSerialization; -import feast.proto.core.IngestionJobProto; import io.prometheus.client.CollectorRegistry; import java.sql.Connection; import java.sql.SQLException; @@ -31,10 +28,8 @@ import javax.persistence.PersistenceContext; import javax.persistence.Table; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; import org.hibernate.engine.spi.SessionImplementor; import org.junit.jupiter.api.*; import org.springframework.boot.test.context.SpringBootTest; @@ -43,11 +38,10 @@ import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.KafkaContainer; @@ -63,6 +57,7 @@ @ActiveProfiles("it") @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@ContextConfiguration public class BaseIT { @Container public static PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer<>(); @@ -123,22 +118,6 @@ public ConsumerFactory testConsumerFactory() { return new DefaultKafkaConsumerFactory<>( props, new StringDeserializer(), new ByteArrayDeserializer()); } - - @Bean - public KafkaTemplate specAckKafkaTemplate( - FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); - - KafkaTemplate t = - new KafkaTemplate<>( - new DefaultKafkaProducerFactory<>( - props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); - t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsAckTopic()); - return t; - } } /** @@ -168,8 +147,8 @@ public static void cleanTables(EntityManager em) throws SQLException { // retries are needed since truncate require exclusive lock // and that often leads to Deadlock // since SpringApp is still running in another thread - var num_retries = 5; - for (var i = 1; i <= num_retries; i++) { + int num_retries = 5; + for (int i = 1; i <= num_retries; i++) { try { Statement statement = connection.createStatement(); statement.execute(String.format("truncate %s cascade", String.join(", ", tableNames))); diff --git a/core/src/test/java/feast/core/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java similarity index 99% rename from core/src/test/java/feast/core/it/DataGenerator.java rename to common-test/src/main/java/feast/common/it/DataGenerator.java index 27133c54cf3..17da60b0cf8 100644 --- a/core/src/test/java/feast/core/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.it; +package feast.common.it; import com.google.common.collect.ImmutableList; import feast.proto.core.FeatureSetProto; diff --git a/core/src/test/java/feast/core/it/SimpleAPIClient.java b/common-test/src/main/java/feast/common/it/SimpleAPIClient.java similarity index 98% rename from core/src/test/java/feast/core/it/SimpleAPIClient.java rename to common-test/src/main/java/feast/common/it/SimpleAPIClient.java index 6e6f66b1019..c9855facdf1 100644 --- a/core/src/test/java/feast/core/it/SimpleAPIClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleAPIClient.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.it; +package feast.common.it; import feast.proto.core.*; import java.util.Arrays; @@ -142,7 +142,7 @@ public List listIngestionJobs() { public String getFeastCoreVersion() { return stub.getFeastCoreVersion( - feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance()) + CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance()) .getVersion(); } diff --git a/core/src/test/java/feast/core/it/SimpleIT.java b/common-test/src/main/java/feast/common/it/SimpleIT.java similarity index 97% rename from core/src/test/java/feast/core/it/SimpleIT.java rename to common-test/src/main/java/feast/common/it/SimpleIT.java index b9e866cf1b0..e8c5dff38c9 100644 --- a/core/src/test/java/feast/core/it/SimpleIT.java +++ b/common-test/src/main/java/feast/common/it/SimpleIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.it; +package feast.common.it; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; diff --git a/common-test/src/main/java/feast/common/util/TestUtil.java b/common-test/src/main/java/feast/common/util/TestUtil.java new file mode 100644 index 00000000000..1931de6a5dc --- /dev/null +++ b/common-test/src/main/java/feast/common/util/TestUtil.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.common.util; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import feast.common.logging.AuditLogger; +import feast.common.logging.config.LoggingProperties; +import org.springframework.boot.info.BuildProperties; + +public class TestUtil { + /** Setup the audit logger. This call is required to use the audit logger when testing. */ + public static void setupAuditLogger() { + LoggingProperties.AuditLogProperties properties = new LoggingProperties.AuditLogProperties(); + properties.setEnabled(true); + LoggingProperties loggingProperties = new LoggingProperties(); + loggingProperties.setAudit(properties); + + BuildProperties buildProperties = mock(BuildProperties.class); + when(buildProperties.getArtifact()).thenReturn("feast-core"); + when(buildProperties.getVersion()).thenReturn("0.6"); + + new AuditLogger(loggingProperties, buildProperties); + } +} diff --git a/common/pom.xml b/common/pom.xml index e38f9d6fcd3..efff48bd04e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -103,13 +103,19 @@ - junit - junit + org.hamcrest + hamcrest-library test - org.hamcrest - hamcrest-library + org.apache.kafka + kafka-clients + 2.5.0 + + + junit + junit + 4.0 test diff --git a/common/src/main/java/feast/common/models/FeatureSetReference.java b/common/src/main/java/feast/common/models/FeatureSetReference.java index 2717e75fa74..42722d58dc8 100644 --- a/common/src/main/java/feast/common/models/FeatureSetReference.java +++ b/common/src/main/java/feast/common/models/FeatureSetReference.java @@ -48,6 +48,20 @@ public static FeatureSetReference of(String projectName, String featureSetName) return FeatureSetReference.of(projectName, featureSetName, -1); } + public static FeatureSetReference parse(String reference) { + String[] split = reference.split("/", 2); + if (split.length == 1) { + return FeatureSetReference.of(PROJECT_DEFAULT_NAME, split[0]); + } + + if (split.length > 2) { + throw new RuntimeException( + "FeatureSet reference must have the format /"); + } + + return FeatureSetReference.of(split[0], split[1]); + } + public String getReference() { return String.format("%s/%s", getProjectName(), getFeatureSetName()); } diff --git a/core/src/main/java/feast/core/util/KafkaSerialization.java b/common/src/main/java/feast/common/util/KafkaSerialization.java similarity index 98% rename from core/src/main/java/feast/core/util/KafkaSerialization.java rename to common/src/main/java/feast/common/util/KafkaSerialization.java index 881d8eb9cf8..62c31a63bb7 100644 --- a/core/src/main/java/feast/core/util/KafkaSerialization.java +++ b/common/src/main/java/feast/common/util/KafkaSerialization.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.util; +package feast.common.util; import com.google.protobuf.GeneratedMessageV3; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/core/pom.xml b/core/pom.xml index e7ddb1a26dc..80e201472ca 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -26,7 +26,7 @@ Feast Core - Feature registry and ingestion coordinator + Feature registry feast-core @@ -91,6 +91,12 @@ feast-auth ${project.version} + + dev.feast + feast-common-test + ${project.version} + test + diff --git a/core/src/main/java/feast/core/config/FeastProperties.java b/core/src/main/java/feast/core/config/FeastProperties.java index 04c5918c210..70cc79f7f36 100644 --- a/core/src/main/java/feast/core/config/FeastProperties.java +++ b/core/src/main/java/feast/core/config/FeastProperties.java @@ -22,13 +22,6 @@ import feast.common.logging.config.LoggingProperties; import feast.common.validators.OneOfStrings; import feast.core.config.FeastProperties.StreamProperties.FeatureStreamOptions; -import feast.proto.core.StoreProto; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Set; import javax.annotation.PostConstruct; import javax.validation.ConstraintViolation; @@ -71,9 +64,6 @@ public FeastProperties() {} /* Feast Core Build Version */ @NotBlank private String version = "unknown"; - /* Population job properties */ - @NotNull private JobProperties jobs; - @NotNull /* Feast Kafka stream properties */ private StreamProperties stream; @@ -93,111 +83,6 @@ LoggingProperties loggingProperties() { return getLogging(); } - /** Feast job properties. These properties are used for ingestion jobs. */ - @Getter - @Setter - public static class JobProperties { - /* Toggle for enabling/disabling job management */ - private Boolean enabled = true; - - @NotBlank - /* The active Apache Beam runner name. This name references one instance of the Runner class */ - private String activeRunner; - - /* Job Coordinator related properties */ - private CoordinatorProperties coordinator; - - @Getter - @Setter - public static class CoordinatorProperties { - /* If true only one IngestionJob would be created per source with all subscribed stores in it */ - private Boolean consolidateJobsPerSource = false; - - /* Labels to identify jobs managed by this job coordinator */ - private Map jobSelector = new HashMap<>(); - - /* Selectors to define featureSets that are responsibility of current JobManager */ - private List featureSetSelector = new ArrayList<>(); - - /* Specify names of stores that must be used by current JobManager */ - private List whitelistedStores = new ArrayList<>(); - - /** - * Similarly to Store's subscription this selector defines set of FeatureSets. All FeatureSets - * that match both project and name will be selected. Project and name may use * - */ - @Getter - @Setter - public static class FeatureSetSelector { - private String project; - private String name; - - public StoreProto.Store.Subscription toSubscription() { - return StoreProto.Store.Subscription.newBuilder() - .setName(this.name) - .setProject(this.project) - .build(); - } - } - } - - /** List of configured job runners. */ - private List runners = new ArrayList<>(); - - /** - * Gets a {@link Runner} instance of the active runner - * - * @return the active runner - */ - public Runner getActiveRunner() { - for (Runner runner : getRunners()) { - if (activeRunner.equals(runner.getName())) { - return runner; - } - } - throw new RuntimeException( - String.format( - "Active runner is misconfigured. Could not find runner: %s.", activeRunner)); - } - - /** Job Runner class. */ - @Getter - @Setter - public static class Runner { - - /** Job runner name. This must be unique. */ - String name; - - /** Job runner type DirectRunner, DataflowRunner currently supported */ - String type; - - /** - * Job runner configuration options. See the following for options - * https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner - */ - Map options = new HashMap<>(); - - /** - * Gets the job runner type as an enum. - * - * @return Returns the job runner type as {@link feast.core.job.Runner} - */ - public feast.core.job.Runner getType() { - return feast.core.job.Runner.fromString(type); - } - } - - @NotNull - /* Population job metric properties */ - private MetricsProperties metrics; - - /* Timeout in seconds for each attempt to update or submit a new job to the runner */ - @Positive private long jobUpdateTimeoutSeconds; - - /* Job update polling interval in millisecond. How frequently Feast will update running jobs. */ - @Positive private long pollingIntervalMilliseconds; - } - /** Properties used to configure Feast's managed Kafka feature stream. */ @Getter @Setter @@ -211,10 +96,6 @@ public static class StreamProperties { /* Feature stream options */ @NotNull private FeatureStreamOptions options; - /* FeatureSetSpec stream options - communication channel between SpecService and IngestionJob - * to update Spec inside job w/o restart */ - @NotNull private FeatureSetSpecStreamProperties specsOptions; - /** Feature stream options */ @Getter @Setter @@ -235,40 +116,6 @@ public static class FeatureStreamOptions { /* Number of Kafka partitions to to use for managed feature stream. */ @Positive private int partitions = 1; } - - @Getter - @Setter - public static class FeatureSetSpecStreamProperties { - /* Kafka topic to send feature set spec to ingestion streaming job */ - @NotBlank private String specsTopic = "feast-feature-set-specs"; - - /* Kafka topic to receive acknowledgment from ingestion job on successful processing of new specs */ - @NotBlank private String specsAckTopic = "feast-feature-set-specs-ack"; - - /* Notify jobs interval in millisecond. - How frequently Feast will check on Pending FeatureSets and publish them to kafka. */ - @Positive private long notifyIntervalMilliseconds; - } - } - - /** Feast population job metrics */ - @Getter - @Setter - public static class MetricsProperties { - - /* Population job metrics enabled */ - private boolean enabled; - - /* Metric type. Possible options: statsd */ - @OneOfStrings({"statsd"}) - @NotBlank - private String type; - - /* Host of metric sink */ - private String host; - - /* Port of metric sink */ - @Positive private int port; } /** @@ -300,33 +147,6 @@ public void validate() { throw new ConstraintViolationException(featureStreamOptionsViolations); } - // Validate JobProperties - Set> jobPropertiesViolations = validator.validate(getJobs()); - if (!jobPropertiesViolations.isEmpty()) { - throw new ConstraintViolationException(jobPropertiesViolations); - } - - // Validate MetricsProperties - if (getJobs().getMetrics().isEnabled()) { - Set> jobMetricViolations = - validator.validate(getJobs().getMetrics()); - if (!jobMetricViolations.isEmpty()) { - throw new ConstraintViolationException(jobMetricViolations); - } - // Additional custom check for hostname value because there is no built-in Spring annotation - // to validate the value is a DNS resolvable hostname or an IP address. - try { - //noinspection ResultOfMethodCallIgnored - InetAddress.getByName(getJobs().getMetrics().getHost()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Invalid config value for feast.jobs.metrics.host: " - + getJobs().getMetrics().getHost() - + ". Make sure it is a valid IP address or DNS hostname e.g. localhost or 10.128.10.40. Error detail: " - + e.getMessage()); - } - } - // Validate AuthenticationProperties Set> authenticationPropsViolations = validator.validate(getSecurity().getAuthentication()); diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java index 6130e0b23bd..cc5e707964f 100644 --- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java +++ b/core/src/main/java/feast/core/config/FeatureStreamConfig.java @@ -18,155 +18,18 @@ import feast.core.config.FeastProperties.StreamProperties; import feast.core.model.Source; -import feast.core.util.KafkaSerialization; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.IngestionJobProto; import feast.proto.core.SourceProto; import feast.proto.core.SourceProto.KafkaSourceConfig; import feast.proto.core.SourceProto.SourceType; -import java.util.HashMap; -import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.config.TopicConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerContainerFactory; -import org.springframework.kafka.core.*; -import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; @Slf4j @Configuration public class FeatureStreamConfig { - String DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG = "15000"; - int DEFAULT_SPECS_TOPIC_PARTITIONING = 1; - short DEFAULT_SPECS_TOPIC_REPLICATION = 1; - - @Bean - public KafkaAdmin admin(FeastProperties feastProperties) { - String bootstrapServers = feastProperties.getStream().getOptions().getBootstrapServers(); - - Map configs = new HashMap<>(); - configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - configs.put( - AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG); - return new KafkaAdmin(configs); - } - - @Bean - public NewTopic featureRowsTopic(FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - - return new NewTopic( - streamProperties.getOptions().getTopic(), - streamProperties.getOptions().getPartitions(), - streamProperties.getOptions().getReplicationFactor()); - } - - @Bean - public NewTopic featureSetSpecsTopic(FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - Map configs = new HashMap<>(); - configs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT); - - NewTopic topic = - new NewTopic( - streamProperties.getSpecsOptions().getSpecsTopic(), - DEFAULT_SPECS_TOPIC_PARTITIONING, - DEFAULT_SPECS_TOPIC_REPLICATION); - - topic.configs(configs); - return topic; - } - - @Bean - public NewTopic featureSetSpecsAckTopic(FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - - return new NewTopic( - streamProperties.getSpecsOptions().getSpecsAckTopic(), - DEFAULT_SPECS_TOPIC_PARTITIONING, - (short) 1); - } - - /** - * Creates kafka publisher for sending FeatureSetSpecs to ingestion job. Uses ProtoSerializer to - * serialize FeatureSetSpec. - * - * @param feastProperties - * @return - */ - @Bean - public KafkaTemplate specKafkaTemplate( - FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - streamProperties.getOptions().getBootstrapServers()); - - KafkaTemplate t = - new KafkaTemplate<>( - new DefaultKafkaProducerFactory<>( - props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); - t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsTopic()); - return t; - } - - /** - * Set configured consumerFactory for specs acknowledgment topic (see ackConsumerFactory) as - * default for KafkaListener. - * - * @param consumerFactory - * @return - */ - @Bean - KafkaListenerContainerFactory< - ConcurrentMessageListenerContainer> - kafkaAckListenerContainerFactory( - ConsumerFactory consumerFactory) { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - /** - * Prepares kafka consumer (by configuring ConsumerFactory) to receive acknowledgments from - * IngestionJob on successful updates of FeatureSetSpecs. - * - * @param feastProperties - * @return ConsumerFactory for FeatureSetSpecAck - */ - @Bean - public ConsumerFactory ackConsumerFactory( - FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - streamProperties.getOptions().getBootstrapServers()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - String.format( - "core-service-%s", FeatureStreamConfig.class.getPackage().getImplementationVersion())); - - return new DefaultKafkaConsumerFactory<>( - props, - new StringDeserializer(), - new KafkaSerialization.ProtoDeserializer<>(IngestionJobProto.FeatureSetSpecAck.parser())); - } - @Autowired @Bean public Source getDefaultSource(FeastProperties feastProperties) { diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 3a7819907db..ab540c08949 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -16,7 +16,6 @@ */ package feast.core.grpc; -import com.google.api.gax.rpc.InvalidArgumentException; import com.google.protobuf.InvalidProtocolBufferException; import feast.auth.service.AuthorizationService; import feast.common.interceptors.GrpcMessageInterceptor; @@ -24,7 +23,6 @@ import feast.core.exception.RetrievalException; import feast.core.grpc.interceptors.MonitoringInterceptor; import feast.core.model.Project; -import feast.core.service.JobService; import feast.core.service.ProjectService; import feast.core.service.SpecService; import feast.core.service.StatsService; @@ -35,7 +33,6 @@ import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import java.util.List; -import java.util.NoSuchElementException; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; @@ -50,7 +47,7 @@ public class CoreServiceImpl extends CoreServiceImplBase { private final FeastProperties feastProperties; private SpecService specService; - private JobService jobService; + // private JobService jobService; private StatsService statsService; private ProjectService projectService; private final AuthorizationService authorizationService; @@ -60,12 +57,12 @@ public CoreServiceImpl( SpecService specService, ProjectService projectService, StatsService statsService, - JobService jobService, + // JobService jobService, FeastProperties feastProperties, AuthorizationService authorizationService) { this.specService = specService; this.projectService = projectService; - this.jobService = jobService; + // this.jobService = jobService; this.feastProperties = feastProperties; this.statsService = statsService; this.authorizationService = authorizationService; @@ -311,68 +308,71 @@ public void listProjects( } } - @Override - public void listIngestionJobs( - ListIngestionJobsRequest request, - StreamObserver responseObserver) { - try { - ListIngestionJobsResponse response = this.jobService.listJobs(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (InvalidArgumentException e) { - log.error("Recieved an invalid request on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void restartIngestionJob( - RestartIngestionJobRequest request, - StreamObserver responseObserver) { - try { - RestartIngestionJobResponse response = this.jobService.restartJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } + // @Override + // public void listIngestionJobs( + // ListIngestionJobsRequest request, + // StreamObserver responseObserver) { + // try { + // ListIngestionJobsResponse response = this.jobService.listJobs(request); + // responseObserver.onNext(response); + // responseObserver.onCompleted(); + // } catch (InvalidArgumentException e) { + // log.error("Recieved an invalid request on calling listIngestionJobs method:", e); + // responseObserver.onError( + // Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); + // } catch (Exception e) { + // log.error("Unexpected exception on calling listIngestionJobs method:", e); + // responseObserver.onError( + // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + // } + // } + // + // @Override + // public void restartIngestionJob( + // RestartIngestionJobRequest request, + // StreamObserver responseObserver) { + // try { + // RestartIngestionJobResponse response = this.jobService.restartJob(request); + // responseObserver.onNext(response); + // responseObserver.onCompleted(); + // } catch (NoSuchElementException e) { + // log.error( + // "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); + // responseObserver.onError( + // Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + // } catch (UnsupportedOperationException e) { + // log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); + // responseObserver.onError( + // + // Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + // } catch (Exception e) { + // log.error("Unexpected exception on calling restartIngestionJob method:", e); + // responseObserver.onError( + // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + // } + // } - @Override - public void stopIngestionJob( - StopIngestionJobRequest request, StreamObserver responseObserver) { - try { - StopIngestionJobResponse response = this.jobService.stopJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } + // @Override + // public void stopIngestionJob( + // StopIngestionJobRequest request, StreamObserver + // responseObserver) { + // try { + // StopIngestionJobResponse response = this.jobService.stopJob(request); + // responseObserver.onNext(response); + // responseObserver.onCompleted(); + // } catch (NoSuchElementException e) { + // log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); + // responseObserver.onError( + // Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + // } catch (UnsupportedOperationException e) { + // log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); + // responseObserver.onError( + // + // Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + // } catch (Exception e) { + // log.error("Unexpected exception on calling stopIngestionJob method:", e); + // responseObserver.onError( + // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + // } + // } } diff --git a/core/src/main/resources/application.yml b/core/src/main/resources/application.yml index 28e8e3f7a72..4f60c3cd03f 100644 --- a/core/src/main/resources/application.yml +++ b/core/src/main/resources/application.yml @@ -16,76 +16,6 @@ # feast: - jobs: - # Enabling JobManagement - enabled: true - - # Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner. - polling_interval_milliseconds: 60000 - - # Timeout in seconds for each attempt to update or submit a new job to the runner. - job_update_timeout_seconds: 240 - - # Name of the active runner in "runners" that should be used. Only a single runner can be active at one time. - active_runner: direct - - # List of runner configurations. Please see protos/feast/core/Runner.proto for more details - # Alternatively see the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner - runners: - - name: direct - type: DirectRunner - options: - tempLocation: gs://bucket/tempLocation - - - name: dataflow - type: DataflowRunner - options: - project: my_gcp_project - region: asia-east1 - workerZone: asia-east1-a - tempLocation: gs://bucket/tempLocation - network: default - subnetwork: regions/asia-east1/subnetworks/mysubnetwork - maxNumWorkers: 1 - enableStreamingEngine: false - workerDiskType: compute.googleapis.com/projects/asia-east1-a/diskTypes/pd-ssd - autoscalingAlgorithm: THROUGHPUT_BASED - usePublicIps: false - workerMachineType: n1-standard-1 - deadLetterTableSpec: project_id:dataset_id.table_id - - # Configuration options for metric collection for all ingestion jobs - metrics: - # Enable metrics pushing for all ingestion jobs. - enabled: false - # Type of metrics sink. Only statsd is currently supported. - type: statsd - # Host of the metrics sink. - host: localhost - # Port of the metrics sink. - port: 9125 - - coordinator: - # if true one job per source with many stores would be created - # if false one job per source-store pair would be created - consolidate-jobs-per-source: false - - # labels (map) that being assigned to job on creation. - # And also used to determine jobs that are being managed by current application - # among all running jobs - jobSelector: - application: feast - - # Specify feature sets that should be handled by current instance of JobManager - featureSetSelector: - - project: "*" - name: "*" - # Stores names that are enabled on current instance of JobManager - whitelisted-stores: - - online - - online_cluster - - historical - stream: # Feature stream type. Only kafka is supported. type: kafka diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java index 84069598b3d..262012d0922 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java @@ -35,7 +35,6 @@ import feast.core.model.Feature; import feast.core.model.FeatureSet; import feast.core.model.Source; -import feast.core.service.JobService; import feast.core.service.ProjectService; import feast.core.service.SpecService; import feast.core.service.StatsService; @@ -68,7 +67,7 @@ public class CoreServiceAuthTest { @Mock private ProjectRepository projectRepository; @Mock private AuthorizationProvider authProvider; @Mock private StatsService statsService; - @Mock private JobService jobService; + // @Mock private JobService jobService; public CoreServiceAuthTest() { MockitoAnnotations.initMocks(this); @@ -84,7 +83,12 @@ public CoreServiceAuthTest() { new AuthorizationService(feastProperties.getSecurity(), authProvider); coreService = new CoreServiceImpl( - specService, projectService, statsService, jobService, feastProperties, authService); + specService, + projectService, + statsService, + // jobService, + feastProperties, + authService); } @Test diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index 71b32ba598a..78995aa8061 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -22,11 +22,11 @@ import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.jwk.JWKSet; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; +import feast.common.it.SimpleAPIClient; import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.it.SimpleAPIClient; import feast.core.model.*; import feast.proto.core.*; import io.grpc.CallCredentials; diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java index 073b9605a8a..ee27b72fa46 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java @@ -24,11 +24,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.jwk.JWKSet; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; +import feast.common.it.SimpleAPIClient; import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.it.SimpleAPIClient; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.FeatureSetProto; import io.grpc.CallCredentials; diff --git a/core/src/test/java/feast/core/metrics/CoreMetricsIT.java b/core/src/test/java/feast/core/metrics/CoreMetricsIT.java index f195ff37a37..f38dc75e815 100644 --- a/core/src/test/java/feast/core/metrics/CoreMetricsIT.java +++ b/core/src/test/java/feast/core/metrics/CoreMetricsIT.java @@ -21,7 +21,7 @@ import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Request; import com.squareup.okhttp.Response; -import feast.core.it.BaseIT; +import feast.common.it.BaseIT; import java.io.IOException; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 8bd935febe4..66d1fb425aa 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -30,9 +30,9 @@ import avro.shaded.com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.it.SimpleAPIClient; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; +import feast.common.it.SimpleAPIClient; import feast.proto.core.*; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; diff --git a/core/src/test/java/feast/core/util/TestUtil.java b/core/src/test/java/feast/core/util/TestUtil.java deleted file mode 100644 index 62d33f31ebe..00000000000 --- a/core/src/test/java/feast/core/util/TestUtil.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import feast.common.logging.AuditLogger; -import feast.common.logging.config.LoggingProperties; -import feast.common.logging.config.LoggingProperties.AuditLogProperties; -import feast.core.model.Entity; -import feast.core.model.Feature; -import feast.core.model.FeatureSet; -import feast.core.model.FeatureSetDeliveryStatus; -import feast.core.model.Project; -import feast.core.model.Source; -import feast.core.model.Store; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetMeta; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import feast.proto.types.ValueProto; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.springframework.boot.info.BuildProperties; - -public class TestUtil { - public static Set makeFeatureSetJobStatus(FeatureSet... featureSets) { - return Stream.of(featureSets) - .map( - fs -> { - FeatureSetDeliveryStatus s = new FeatureSetDeliveryStatus(); - return s; - }) - .collect(Collectors.toSet()); - } - - public static Set makeFeatureSetJobStatus( - List featureSets) { - return makeFeatureSetJobStatus(featureSets.toArray(FeatureSet[]::new)); - } - - public static Source createKafkaSource(String boostrapServers, String topic, boolean isDefault) { - return Source.fromProto( - SourceProto.Source.newBuilder() - .setType(SourceProto.SourceType.KAFKA) - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(boostrapServers) - .setTopic(topic) - .build()) - .build(), - true); - } - - public static Source defaultSource = createKafkaSource("kafka:9092", "topic", true); - - public static Store createStore(String name, List subscriptions) { - return Store.fromProto( - StoreProto.Store.newBuilder() - .setName(name) - .setType(StoreType.REDIS) - .setRedisConfig(RedisConfig.newBuilder().build()) - .addAllSubscriptions(subscriptions) - .build()); - } - - public static FeatureSet CreateFeatureSet( - String name, String project, List entities, List features) { - FeatureSet fs = - new FeatureSet( - name, - project, - 100L, - entities, - features, - defaultSource, - new HashMap<>(), - FeatureSetProto.FeatureSetStatus.STATUS_READY); - fs.setVersion(1); - return fs; - } - - public static FeatureSet createEmptyFeatureSet(String name, Source source) { - return FeatureSet.fromProto( - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setSource(source.toProto()) - .setProject(Project.DEFAULT_NAME) - .setName(name)) - .setMeta(FeatureSetMeta.newBuilder().build()) - .build()); - } - - public static Feature CreateFeature(String name, ValueProto.ValueType.Enum valueType) { - return CreateFeature(name, valueType, Map.of()); - } - - public static Feature CreateFeature( - String name, ValueProto.ValueType.Enum valueType, Map labels) { - return Feature.fromProto( - FeatureSetProto.FeatureSpec.newBuilder() - .setName(name) - .setValueType(valueType) - .putAllLabels(labels) - .build()); - } - - public static Entity CreateEntity(String name, ValueProto.ValueType.Enum valueType) { - return Entity.fromProto( - FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build()); - } - - /** Setup the audit logger. This call is required to use the audit logger when testing. */ - public static void setupAuditLogger() { - AuditLogProperties properties = new AuditLogProperties(); - properties.setEnabled(true); - LoggingProperties loggingProperties = new LoggingProperties(); - loggingProperties.setAudit(properties); - - BuildProperties buildProperties = mock(BuildProperties.class); - when(buildProperties.getArtifact()).thenReturn("feast-core"); - when(buildProperties.getVersion()).thenReturn("0.6"); - - new AuditLogger(loggingProperties, buildProperties); - } -} diff --git a/core/src/test/resources/application-it.properties b/core/src/test/resources/application-it.properties index ccb339fa9ab..80d1baea473 100644 --- a/core/src/test/resources/application-it.properties +++ b/core/src/test/resources/application-it.properties @@ -21,11 +21,6 @@ feast.security.authorization.enabled = false feast.jobs.enabled=false -spring.jpa.properties.hibernate.format_sql=true -spring.jpa.properties.hibernate.show_sql=false -spring.jpa.hibernate.naming.physical-strategy=org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy -spring.jpa.hibernate.ddl-auto=none - spring.datasource.hikari.maximum-pool-size=100 spring.main.allow-bean-definition-overriding=true diff --git a/job-coordinator/pom.xml b/job-coordinator/pom.xml new file mode 100644 index 00000000000..f45d26dd9c6 --- /dev/null +++ b/job-coordinator/pom.xml @@ -0,0 +1,335 @@ + + + + 4.0.0 + + + dev.feast + feast-parent + ${revision} + + + Feast Job Coordinator + Feature ingestion coordinator + feast-job-coordinator + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + + + + + org.jacoco + jacoco-maven-plugin + + + + org.springframework.boot + spring-boot-maven-plugin + + false + + + + build-info + + build-info + + + + + + + + + + dev.feast + feast-ingestion + ${project.version} + + + + org.slf4j + slf4j-simple + + + + + dev.feast + feast-common + ${project.version} + + + dev.feast + feast-auth + ${project.version} + + + dev.feast + feast-core + ${project.version} + test + + + + + org.springframework.boot + spring-boot-devtools + true + + + + javax.inject + javax.inject + 1 + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-log4j2 + + + org.apache.logging.log4j + log4j-web + + + org.springframework.security + spring-security-core + ${spring.security.version} + + + org.springframework.security + spring-security-config + ${spring.security.version} + + + org.springframework.security.oauth + spring-security-oauth2 + ${spring.security.oauth2.version} + + + org.springframework.security + spring-security-oauth2-client + ${spring.security.version} + + + org.springframework.security + spring-security-web + ${spring.security.version} + + + org.springframework.security + spring-security-oauth2-resource-server + ${spring.security.version} + + + org.springframework.security + spring-security-oauth2-jose + ${spring.security.version} + + + net.devh + grpc-server-spring-boot-starter + ${grpc.spring.boot.starter.version} + + + com.nimbusds + nimbus-jose-jwt + 8.2.1 + + + org.springframework.security + spring-security-oauth2-core + ${spring.security.version} + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + org.springframework.boot + spring-boot-starter-actuator + + + + + org.springframework.boot + spring-boot-configuration-processor + + + + io.grpc + grpc-services + + + + io.grpc + grpc-stub + + + + com.google.protobuf + protobuf-java-util + + + + com.google.guava + guava + + + + com.google.code.gson + gson + 2.8.5 + + + com.google.api-client + google-api-client + 1.30.9 + + + com.google.apis + google-api-services-dataflow + v1b3-rev20200305-1.30.9 + + + + org.springframework.kafka + spring-kafka + + + + + org.projectlombok + lombok + ${lombok.version} + + + + org.hamcrest + hamcrest-library + + + + io.prometheus + simpleclient + + + + io.prometheus + simpleclient_servlet + + + com.google.api.client + google-api-client-googleapis-auth-oauth + 1.2.3-alpha + + + com.auth0 + jwks-rsa + 0.11.0 + + + + com.auth0 + java-jwt + 3.10.0 + + + + + com.jayway.jsonpath + json-path-assert + 2.2.0 + test + + + + org.springframework.boot + spring-boot-test + test + + + org.springframework.boot + spring-boot-test-autoconfigure + test + + + javax.xml.bind + jaxb-api + + + org.mockito + mockito-core + ${mockito.version} + test + + + dev.feast + feast-common-test + ${project.version} + test + + + sh.ory.keto + keto-client + 0.4.4-alpha.1 + test + + + com.github.tomakehurst + wiremock + 2.27.0 + test + + + com.google.auto.value + auto-value-annotations + 1.6.6 + + + com.google.auto.value + auto-value + 1.6.6 + provided + + + + + org.postgresql + postgresql + test + true + + + diff --git a/job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java b/job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java new file mode 100644 index 00000000000..0fca1ff6eb7 --- /dev/null +++ b/job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.jc; + +import feast.jc.config.FeastProperties; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@SpringBootApplication( + exclude = { + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class + }) +@EnableConfigurationProperties(FeastProperties.class) +@Slf4j +public class JobCoordinatorApplication { + public static void main(String[] args) { + SpringApplication.run(JobCoordinatorApplication.class, args); + } +} diff --git a/job-coordinator/src/main/java/feast/jc/config/FeastProperties.java b/job-coordinator/src/main/java/feast/jc/config/FeastProperties.java new file mode 100644 index 00000000000..322692ac3e4 --- /dev/null +++ b/job-coordinator/src/main/java/feast/jc/config/FeastProperties.java @@ -0,0 +1,315 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.jc.config; + +import feast.common.logging.config.LoggingProperties; +import feast.common.validators.OneOfStrings; +import feast.proto.core.StoreProto; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import javax.annotation.PostConstruct; +import javax.validation.*; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import lombok.Getter; +import lombok.Setter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.info.BuildProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +@ComponentScan("feast.common.logging") +@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true) +public class FeastProperties { + + /** + * Instantiates a new Feast properties. + * + * @param buildProperties Feast build properties + */ + @Autowired + public FeastProperties(BuildProperties buildProperties) { + setVersion(buildProperties.getVersion()); + } + + /** Instantiates a new Feast properties. */ + public FeastProperties() {} + + /* Feast Job Coordinator Build Version */ + @NotBlank private String version = "unknown"; + + /* Feast Core Address */ + @NotBlank private String coreHost; + private Integer corePort; + + /* Population job properties */ + @NotNull private JobProperties jobs; + + @NotNull + /* Feast Kafka stream properties */ + private StreamProperties stream; + + /* Feast Audit Logging properties */ + @NotNull private LoggingProperties logging; + + @Bean + LoggingProperties loggingProperties() { + return getLogging(); + } + + /** Feast job properties. These properties are used for ingestion jobs. */ + @Getter + @Setter + public static class JobProperties { + /* Toggle for enabling/disabling job management */ + private Boolean enabled = true; + + @NotBlank + /* The active Apache Beam runner name. This name references one instance of the Runner class */ + private String activeRunner; + + /* Job Coordinator related properties */ + private CoordinatorProperties coordinator; + + @Getter + @Setter + public static class CoordinatorProperties { + /* If true only one IngestionJob would be created per source with all subscribed stores in it */ + private Boolean consolidateJobsPerSource = false; + + /* Labels to identify jobs managed by this job coordinator */ + private Map jobSelector = new HashMap<>(); + + /* Selectors to define featureSets that are responsibility of current JobManager */ + private List featureSetSelector = new ArrayList<>(); + + /* Specify names of stores that must be used by current JobManager */ + private List whitelistedStores = new ArrayList<>(); + + /** + * Similarly to Store's subscription this selector defines set of FeatureSets. All FeatureSets + * that match both project and name will be selected. Project and name may use * + */ + @Getter + @Setter + public static class FeatureSetSelector { + private String project; + private String name; + + public StoreProto.Store.Subscription toSubscription() { + return StoreProto.Store.Subscription.newBuilder() + .setName(this.name) + .setProject(this.project) + .build(); + } + } + } + + /** List of configured job runners. */ + private List runners = new ArrayList<>(); + + /** + * Gets a {@link Runner} instance of the active runner + * + * @return the active runner + */ + public Runner getActiveRunner() { + for (Runner runner : getRunners()) { + if (activeRunner.equals(runner.getName())) { + return runner; + } + } + throw new RuntimeException( + String.format( + "Active runner is misconfigured. Could not find runner: %s.", activeRunner)); + } + + /** Job Runner class. */ + @Getter + @Setter + public static class Runner { + + /** Job runner name. This must be unique. */ + String name; + + /** Job runner type DirectRunner, DataflowRunner currently supported */ + String type; + + /** + * Job runner configuration options. See the following for options + * https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner + */ + Map options = new HashMap<>(); + + /** + * Gets the job runner type as an enum. + * + * @return Returns the job runner type as {@link feast.jc.runner.Runner} + */ + public feast.jc.runner.Runner getType() { + return feast.jc.runner.Runner.fromString(type); + } + } + + @NotNull + /* Population job metric properties */ + private MetricsProperties metrics; + + /* Timeout in seconds for each attempt to update or submit a new job to the runner */ + @Positive private long jobUpdateTimeoutSeconds; + + /* Job update polling interval in millisecond. How frequently Feast will update running jobs. */ + @Positive private long pollingIntervalMilliseconds; + } + + /** Properties used to configure Feast's managed Kafka feature stream. */ + @Getter + @Setter + public static class StreamProperties { + + /* Feature stream type. Only "kafka" is supported. */ + @OneOfStrings({"kafka"}) + @NotBlank + private String type; + + /* Feature stream options */ + @NotNull private FeatureStreamOptions options; + + /* FeatureSetSpec stream options - communication channel between SpecService and IngestionJob + * to update Spec inside job w/o restart */ + @NotNull private FeatureSetSpecStreamProperties specsOptions; + + /** Feature stream options */ + @Getter + @Setter + public static class FeatureStreamOptions { + + /* Kafka topic to use for feature sets without source topics. */ + @NotBlank private String topic = "feast-features"; + + /** + * Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined + * source. + */ + @NotBlank private String bootstrapServers = "localhost:9092"; + + /* Defines the number of copies of managed feature stream Kafka. */ + @Positive private short replicationFactor = 1; + + /* Number of Kafka partitions to to use for managed feature stream. */ + @Positive private int partitions = 1; + } + + @Getter + @Setter + public static class FeatureSetSpecStreamProperties { + /* Kafka topic to send feature set spec to ingestion streaming job */ + @NotBlank private String specsTopic = "feast-feature-set-specs"; + + /* Kafka topic to receive acknowledgment from ingestion job on successful processing of new specs */ + @NotBlank private String specsAckTopic = "feast-feature-set-specs-ack"; + + /* Notify jobs interval in millisecond. + How frequently Feast will check on Pending FeatureSets and publish them to kafka. */ + @Positive private long notifyIntervalMilliseconds; + } + } + + /** Feast population job metrics */ + @Getter + @Setter + public static class MetricsProperties { + + /* Population job metrics enabled */ + private boolean enabled; + + /* Metric type. Possible options: statsd */ + @OneOfStrings({"statsd"}) + @NotBlank + private String type; + + /* Host of metric sink */ + private String host; + + /* Port of metric sink */ + @Positive private int port; + } + + /** + * Validates all FeastProperties. This method runs after properties have been initialized and + * individually and conditionally validates each class. + */ + @PostConstruct + public void validate() { + ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); + Validator validator = factory.getValidator(); + + // Validate root fields in FeastProperties + Set> violations = validator.validate(this); + if (!violations.isEmpty()) { + throw new ConstraintViolationException(violations); + } + + // Validate Stream properties + Set> streamPropertyViolations = + validator.validate(getStream()); + if (!streamPropertyViolations.isEmpty()) { + throw new ConstraintViolationException(streamPropertyViolations); + } + + // Validate Stream Options + Set> featureStreamOptionsViolations = + validator.validate(getStream().getOptions()); + if (!featureStreamOptionsViolations.isEmpty()) { + throw new ConstraintViolationException(featureStreamOptionsViolations); + } + + // Validate JobProperties + Set> jobPropertiesViolations = validator.validate(getJobs()); + if (!jobPropertiesViolations.isEmpty()) { + throw new ConstraintViolationException(jobPropertiesViolations); + } + + // Validate MetricsProperties + if (getJobs().getMetrics().isEnabled()) { + Set> jobMetricViolations = + validator.validate(getJobs().getMetrics()); + if (!jobMetricViolations.isEmpty()) { + throw new ConstraintViolationException(jobMetricViolations); + } + // Additional custom check for hostname value because there is no built-in Spring annotation + // to validate the value is a DNS resolvable hostname or an IP address. + try { + //noinspection ResultOfMethodCallIgnored + InetAddress.getByName(getJobs().getMetrics().getHost()); + } catch (UnknownHostException e) { + throw new IllegalArgumentException( + "Invalid config value for feast.jobs.metrics.host: " + + getJobs().getMetrics().getHost() + + ". Make sure it is a valid IP address or DNS hostname e.g. localhost or 10.128.10.40. Error detail: " + + e.getMessage()); + } + } + } +} diff --git a/job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java b/job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java new file mode 100644 index 00000000000..858047a8f09 --- /dev/null +++ b/job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.jc.config; + +import feast.common.util.KafkaSerialization; +import feast.proto.core.FeatureSetProto; +import feast.proto.core.IngestionJobProto; +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.config.TopicConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.KafkaListenerContainerFactory; +import org.springframework.kafka.core.*; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; + +@Slf4j +@Configuration +public class FeatureStreamConfig { + + String DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG = "15000"; + int DEFAULT_SPECS_TOPIC_PARTITIONING = 1; + short DEFAULT_SPECS_TOPIC_REPLICATION = 1; + + @Bean + public KafkaAdmin admin(FeastProperties feastProperties) { + String bootstrapServers = feastProperties.getStream().getOptions().getBootstrapServers(); + + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + configs.put( + AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG); + return new KafkaAdmin(configs); + } + + @Bean + public NewTopic featureRowsTopic(FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + + return new NewTopic( + streamProperties.getOptions().getTopic(), + streamProperties.getOptions().getPartitions(), + streamProperties.getOptions().getReplicationFactor()); + } + + @Bean + public NewTopic featureSetSpecsTopic(FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + Map configs = new HashMap<>(); + configs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT); + + NewTopic topic = + new NewTopic( + streamProperties.getSpecsOptions().getSpecsTopic(), + DEFAULT_SPECS_TOPIC_PARTITIONING, + DEFAULT_SPECS_TOPIC_REPLICATION); + + topic.configs(configs); + return topic; + } + + @Bean + public NewTopic featureSetSpecsAckTopic(FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + + return new NewTopic( + streamProperties.getSpecsOptions().getSpecsAckTopic(), + DEFAULT_SPECS_TOPIC_PARTITIONING, + (short) 1); + } + + /** + * Creates kafka publisher for sending FeatureSetSpecs to ingestion job. Uses ProtoSerializer to + * serialize FeatureSetSpec. + * + * @param feastProperties + * @return + */ + @Bean + public KafkaTemplate specKafkaTemplate( + FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + Map props = new HashMap<>(); + + props.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + streamProperties.getOptions().getBootstrapServers()); + + KafkaTemplate t = + new KafkaTemplate<>( + new DefaultKafkaProducerFactory<>( + props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); + t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsTopic()); + return t; + } + + /** + * Set configured consumerFactory for specs acknowledgment topic (see ackConsumerFactory) as + * default for KafkaListener. + * + * @param consumerFactory + * @return + */ + @Bean + KafkaListenerContainerFactory< + ConcurrentMessageListenerContainer> + kafkaAckListenerContainerFactory( + ConsumerFactory consumerFactory) { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory); + return factory; + } + + /** + * Prepares kafka consumer (by configuring ConsumerFactory) to receive acknowledgments from + * IngestionJob on successful updates of FeatureSetSpecs. + * + * @param feastProperties + * @return ConsumerFactory for FeatureSetSpecAck + */ + @Bean + public ConsumerFactory ackConsumerFactory( + FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + Map props = new HashMap<>(); + + props.put( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + streamProperties.getOptions().getBootstrapServers()); + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + String.format("core-service-%s", feastProperties.getVersion())); + + return new DefaultKafkaConsumerFactory<>( + props, + new StringDeserializer(), + new KafkaSerialization.ProtoDeserializer<>(IngestionJobProto.FeatureSetSpecAck.parser())); + } +} diff --git a/core/src/main/java/feast/core/config/JobConfig.java b/job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java similarity index 77% rename from core/src/main/java/feast/core/config/JobConfig.java rename to job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java index f6f3375732d..48e812dc8a3 100644 --- a/core/src/main/java/feast/core/config/JobConfig.java +++ b/job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java @@ -14,26 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.config; +package feast.jc.config; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import feast.core.config.FeastProperties.JobProperties; -import feast.core.job.ConsolidatedJobStrategy; -import feast.core.job.JobGroupingStrategy; -import feast.core.job.JobManager; -import feast.core.job.JobPerStoreStrategy; -import feast.core.job.JobRepository; -import feast.core.job.dataflow.DataflowJobManager; -import feast.core.job.direct.DirectJobRegistry; -import feast.core.job.direct.DirectRunnerJobManager; +import feast.jc.dao.JobRepository; +import feast.jc.runner.ConsolidatedJobStrategy; +import feast.jc.runner.JobGroupingStrategy; +import feast.jc.runner.JobManager; +import feast.jc.runner.JobPerStoreStrategy; +import feast.jc.runner.dataflow.DataflowJobManager; +import feast.jc.runner.direct.DirectJobRegistry; +import feast.jc.runner.direct.DirectRunnerJobManager; +import feast.proto.core.CoreServiceGrpc; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import feast.proto.core.SourceProto; +import io.grpc.CallCredentials; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,7 +45,7 @@ /** Beans for job management */ @Slf4j @Configuration -public class JobConfig { +public class JobCoordinatorConfig { private final Gson gson = new Gson(); /** @@ -101,7 +105,7 @@ public JobManager getJobManager( IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig) throws InvalidProtocolBufferException { - JobProperties jobProperties = feastProperties.getJobs(); + FeastProperties.JobProperties jobProperties = feastProperties.getJobs(); FeastProperties.JobProperties.Runner runner = jobProperties.getActiveRunner(); Map runnerConfigOptions = runner.getOptions(); @@ -131,4 +135,23 @@ public JobManager getJobManager( throw new IllegalArgumentException("Unsupported runner: " + runner); } } + + @Bean + public CoreServiceGrpc.CoreServiceBlockingStub coreService( + FeastProperties feastProperties, ObjectProvider callCredentials) { + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + feastProperties.getCoreHost(), feastProperties.getCorePort()) + .usePlaintext() + .build(); + CallCredentials creds = callCredentials.getIfAvailable(); + + CoreServiceGrpc.CoreServiceBlockingStub blockingStub; + if (creds != null) { + blockingStub = CoreServiceGrpc.newBlockingStub(channel).withCallCredentials(creds); + } else { + blockingStub = CoreServiceGrpc.newBlockingStub(channel); + } + return blockingStub; + } } diff --git a/core/src/main/java/feast/core/job/InMemoryJobRepository.java b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java similarity index 97% rename from core/src/main/java/feast/core/job/InMemoryJobRepository.java rename to job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java index 918bad50ea7..343700db31c 100644 --- a/core/src/main/java/feast/core/job/InMemoryJobRepository.java +++ b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java @@ -14,12 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.dao; import com.google.common.collect.Lists; import feast.common.models.FeatureSetReference; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import feast.proto.core.SourceProto; import java.util.*; import java.util.function.Predicate; diff --git a/core/src/main/java/feast/core/job/JobRepository.java b/job-coordinator/src/main/java/feast/jc/dao/JobRepository.java similarity index 94% rename from core/src/main/java/feast/core/job/JobRepository.java rename to job-coordinator/src/main/java/feast/jc/dao/JobRepository.java index bde3f1f711a..e9737d3863e 100644 --- a/core/src/main/java/feast/core/job/JobRepository.java +++ b/job-coordinator/src/main/java/feast/jc/dao/JobRepository.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.dao; import feast.common.models.FeatureSetReference; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; import feast.proto.core.SourceProto; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/feast/core/exception/JobExecutionException.java b/job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java similarity index 97% rename from core/src/main/java/feast/core/exception/JobExecutionException.java rename to job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java index 85eb199ae47..61f918a13f8 100644 --- a/core/src/main/java/feast/core/exception/JobExecutionException.java +++ b/job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.exception; +package feast.jc.exception; /** Exception thrown when a request for job execution fails. */ public class JobExecutionException extends RuntimeException { diff --git a/core/src/main/java/feast/core/exception/JobMonitoringException.java b/job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java similarity index 97% rename from core/src/main/java/feast/core/exception/JobMonitoringException.java rename to job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java index 380f68a7786..511046b3112 100644 --- a/core/src/main/java/feast/core/exception/JobMonitoringException.java +++ b/job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.exception; +package feast.jc.exception; /** Exception thrown when error happen during job monitoring. */ public class JobMonitoringException extends RuntimeException { diff --git a/core/src/main/java/feast/core/model/FeatureSetDeliveryStatus.java b/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java similarity index 98% rename from core/src/main/java/feast/core/model/FeatureSetDeliveryStatus.java rename to job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java index c0fcd8b573c..850d25ef17d 100644 --- a/core/src/main/java/feast/core/model/FeatureSetDeliveryStatus.java +++ b/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.model; +package feast.jc.model; import com.google.common.base.Objects; import feast.common.models.FeatureSetReference; diff --git a/core/src/main/java/feast/core/model/Job.java b/job-coordinator/src/main/java/feast/jc/model/Job.java similarity index 99% rename from core/src/main/java/feast/core/model/Job.java rename to job-coordinator/src/main/java/feast/jc/model/Job.java index d67d16cba64..c007ea33804 100644 --- a/core/src/main/java/feast/core/model/Job.java +++ b/job-coordinator/src/main/java/feast/jc/model/Job.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.model; +package feast.jc.model; import com.google.auto.value.AutoValue; import feast.common.models.FeatureSetReference; diff --git a/core/src/main/java/feast/core/model/JobStatus.java b/job-coordinator/src/main/java/feast/jc/model/JobStatus.java similarity index 99% rename from core/src/main/java/feast/core/model/JobStatus.java rename to job-coordinator/src/main/java/feast/jc/model/JobStatus.java index 6bafc06ec90..25a65892ea9 100644 --- a/core/src/main/java/feast/core/model/JobStatus.java +++ b/job-coordinator/src/main/java/feast/jc/model/JobStatus.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.model; +package feast.jc.model; import feast.proto.core.IngestionJobProto.IngestionJobStatus; import java.util.Map; diff --git a/core/src/main/java/feast/core/job/ConsolidatedJobStrategy.java b/job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java similarity index 96% rename from core/src/main/java/feast/core/job/ConsolidatedJobStrategy.java rename to job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java index a8b1c5d2744..1f9e302e312 100644 --- a/core/src/main/java/feast/core/job/ConsolidatedJobStrategy.java +++ b/job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.dao.JobRepository; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.time.Instant; diff --git a/core/src/main/java/feast/core/job/JobGroupingStrategy.java b/job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java similarity index 96% rename from core/src/main/java/feast/core/job/JobGroupingStrategy.java rename to job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java index a8cd0d3cd7a..34d2df3c7a6 100644 --- a/core/src/main/java/feast/core/job/JobGroupingStrategy.java +++ b/job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; -import feast.core.model.Job; +import feast.jc.model.Job; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.util.Map; diff --git a/core/src/main/java/feast/core/job/JobManager.java b/job-coordinator/src/main/java/feast/jc/runner/JobManager.java similarity index 95% rename from core/src/main/java/feast/core/job/JobManager.java rename to job-coordinator/src/main/java/feast/jc/runner/JobManager.java index ffae10b3077..06891761260 100644 --- a/core/src/main/java/feast/core/job/JobManager.java +++ b/job-coordinator/src/main/java/feast/jc/runner/JobManager.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; import java.util.List; public interface JobManager { diff --git a/core/src/main/java/feast/core/job/JobPerStoreStrategy.java b/job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java similarity index 96% rename from core/src/main/java/feast/core/job/JobPerStoreStrategy.java rename to job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java index f967cefac08..17d6874bf93 100644 --- a/core/src/main/java/feast/core/job/JobPerStoreStrategy.java +++ b/job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; import com.google.common.collect.Lists; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.dao.JobRepository; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.time.Instant; diff --git a/core/src/main/java/feast/core/job/Runner.java b/job-coordinator/src/main/java/feast/jc/runner/Runner.java similarity index 98% rename from core/src/main/java/feast/core/job/Runner.java rename to job-coordinator/src/main/java/feast/jc/runner/Runner.java index acccb70c8b2..6a5f3449127 100644 --- a/core/src/main/java/feast/core/job/Runner.java +++ b/job-coordinator/src/main/java/feast/jc/runner/Runner.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; import java.util.NoSuchElementException; diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java similarity index 95% rename from core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java rename to job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java index 1506905c7ac..215156e2bdf 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java +++ b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java @@ -14,12 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; -import static feast.core.util.PipelineUtil.detectClassPathResourcesToStage; -import static feast.core.util.StreamUtil.wrapException; import static feast.ingestion.utils.SpecUtil.parseSourceJson; import static feast.ingestion.utils.SpecUtil.parseStoreJsonList; +import static feast.jc.util.PipelineUtil.detectClassPathResourcesToStage; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; @@ -30,13 +29,15 @@ import com.google.common.base.Strings; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import feast.core.config.FeastProperties.MetricsProperties; -import feast.core.exception.JobExecutionException; -import feast.core.job.JobManager; -import feast.core.job.Runner; -import feast.core.model.*; +import feast.common.models.FeatureSetReference; import feast.ingestion.ImportJob; import feast.ingestion.options.ImportOptions; +import feast.jc.config.FeastProperties.MetricsProperties; +import feast.jc.exception.JobExecutionException; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; +import feast.jc.runner.Runner; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import feast.proto.core.SourceProto; @@ -340,14 +341,18 @@ private ImportOptions getPipelineOptions( PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class); JsonFormat.Printer jsonPrinter = JsonFormat.printer(); + List storesJson = new ArrayList<>(); + for (StoreProto.Store sink : sinks) { + String print = jsonPrinter.print(sink); + storesJson.add(print); + } pipelineOptions.setSpecsStreamingUpdateConfigJson( jsonPrinter.print(specsStreamingUpdateConfig)); pipelineOptions.setSourceJson(jsonPrinter.print(source)); - pipelineOptions.setStoresJson( - sinks.stream().map(wrapException(jsonPrinter::print)).collect(Collectors.toList())); + pipelineOptions.setStoresJson(storesJson); pipelineOptions.setProject(projectId); - pipelineOptions.setDefaultFeastProject(Project.DEFAULT_NAME); + pipelineOptions.setDefaultFeastProject(FeatureSetReference.PROJECT_DEFAULT_NAME); pipelineOptions.setUpdate(update); pipelineOptions.setRunner(DataflowRunner.class); pipelineOptions.setJobName(jobName); diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobState.java b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java similarity index 96% rename from core/src/main/java/feast/core/job/dataflow/DataflowJobState.java rename to job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java index a5da7fd9798..49992208828 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobState.java +++ b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; public enum DataflowJobState { JOB_STATE_UNKNOWN, diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobStateMapper.java b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java similarity index 72% rename from core/src/main/java/feast/core/job/dataflow/DataflowJobStateMapper.java rename to job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java index ec5738be699..0f46652f895 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobStateMapper.java +++ b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java @@ -14,21 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_CANCELLED; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_CANCELLING; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_DONE; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_DRAINED; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_DRAINING; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_FAILED; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_PENDING; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_RUNNING; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_STOPPED; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_UNKNOWN; -import static feast.core.job.dataflow.DataflowJobState.JOB_STATE_UPDATED; +import static feast.jc.runner.dataflow.DataflowJobState.*; -import feast.core.model.JobStatus; +import feast.jc.model.JobStatus; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobType.java b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java similarity index 95% rename from core/src/main/java/feast/core/job/dataflow/DataflowJobType.java rename to job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java index e96bbd2141e..578e7c7cd46 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobType.java +++ b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; public enum DataflowJobType { JOB_TYPE_BATCH, diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java similarity index 97% rename from core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java rename to job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java index a87afa1bcf6..d723ca2b128 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java +++ b/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; -import feast.core.job.option.RunnerConfig; +import feast.jc.runner.option.RunnerConfig; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import java.util.*; +import java.util.Map; +import java.util.Set; import javax.validation.*; import javax.validation.constraints.NotBlank; import lombok.Getter; diff --git a/core/src/main/java/feast/core/job/direct/DirectJob.java b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java similarity index 97% rename from core/src/main/java/feast/core/job/direct/DirectJob.java rename to job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java index 35c778f360c..600572581ac 100644 --- a/core/src/main/java/feast/core/job/direct/DirectJob.java +++ b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; import java.io.IOException; import lombok.AllArgsConstructor; diff --git a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java similarity index 98% rename from core/src/main/java/feast/core/job/direct/DirectJobRegistry.java rename to job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java index f7ded9fec76..da3567f0d84 100644 --- a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java +++ b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; import com.google.common.base.Strings; import java.io.IOException; diff --git a/core/src/main/java/feast/core/job/direct/DirectJobStateMapper.java b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java similarity index 96% rename from core/src/main/java/feast/core/job/direct/DirectJobStateMapper.java rename to job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java index dd1c81d83ef..dd649a4d08f 100644 --- a/core/src/main/java/feast/core/job/direct/DirectJobStateMapper.java +++ b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; -import feast.core.model.JobStatus; +import feast.jc.model.JobStatus; import java.util.HashMap; import java.util.Map; import org.apache.beam.sdk.PipelineResult.State; diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java similarity index 94% rename from core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java rename to job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java index e48ef15bbed..6e53085a588 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java +++ b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; -import feast.core.job.option.RunnerConfig; +import feast.jc.runner.option.RunnerConfig; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; public class DirectRunnerConfig extends RunnerConfig { diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java similarity index 91% rename from core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java rename to job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java index b02e3dac4fb..4e81394c24e 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java +++ b/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java @@ -14,26 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; - -import static feast.core.util.StreamUtil.wrapException; +package feast.jc.runner.direct; import com.google.common.base.Strings; import com.google.protobuf.util.JsonFormat; -import feast.core.config.FeastProperties.MetricsProperties; -import feast.core.exception.JobExecutionException; -import feast.core.job.JobManager; -import feast.core.job.Runner; -import feast.core.model.*; +import feast.common.models.FeatureSetReference; import feast.ingestion.ImportJob; import feast.ingestion.options.ImportOptions; +import feast.jc.config.FeastProperties.MetricsProperties; +import feast.jc.exception.JobExecutionException; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; +import feast.jc.runner.Runner; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.io.IOException; import java.util.*; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.beam.runners.direct.DirectRunner; import org.apache.beam.sdk.PipelineResult; @@ -93,13 +92,18 @@ private ImportOptions getPipelineOptions( PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class); JsonFormat.Printer printer = JsonFormat.printer(); + List storesJson = new ArrayList<>(); + for (StoreProto.Store sink : sinks) { + String print = printer.print(sink); + storesJson.add(print); + } + pipelineOptions.setSpecsStreamingUpdateConfigJson(printer.print(specsStreamingUpdateConfig)); pipelineOptions.setSourceJson(printer.print(source)); pipelineOptions.setJobName(jobName); - pipelineOptions.setStoresJson( - sinks.stream().map(wrapException(printer::print)).collect(Collectors.toList())); + pipelineOptions.setStoresJson(storesJson); pipelineOptions.setRunner(DirectRunner.class); - pipelineOptions.setDefaultFeastProject(Project.DEFAULT_NAME); + pipelineOptions.setDefaultFeastProject(FeatureSetReference.PROJECT_DEFAULT_NAME); pipelineOptions.setProject(""); // set to default value to satisfy validation if (metrics.isEnabled()) { pipelineOptions.setMetricsExporterType(metrics.getType()); diff --git a/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java b/job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java similarity index 98% rename from core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java rename to job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java index 2f6b37df1b5..3366527c55d 100644 --- a/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java +++ b/job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.option; +package feast.jc.runner.option; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; diff --git a/core/src/main/java/feast/core/job/option/RunnerConfig.java b/job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java similarity index 92% rename from core/src/main/java/feast/core/job/option/RunnerConfig.java rename to job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java index 44550f9608e..36cd67aba52 100644 --- a/core/src/main/java/feast/core/job/option/RunnerConfig.java +++ b/job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.option; +package feast.jc.runner.option; -import feast.core.util.TypeConversion; +import feast.jc.util.TypeConversion; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -24,8 +24,8 @@ /** * Value class containing the application-default configuration for a runner. When a job is started - * by core, all fields in the object will be converted into --key=value args to seed the beam - * pipeline options. + * by JC, all fields in the object will be converted into --key=value args to seed the beam pipeline + * options. */ public abstract class RunnerConfig { diff --git a/core/src/main/java/feast/core/job/task/CreateJobTask.java b/job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java similarity index 92% rename from core/src/main/java/feast/core/job/task/CreateJobTask.java rename to job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java index 662749b3bc2..d3cef51862a 100644 --- a/core/src/main/java/feast/core/job/task/CreateJobTask.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java @@ -14,17 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.core.job.JobManager; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; -/** Task that starts recently created {@link Job} by using {@link feast.core.job.JobManager}. */ +/** Task that starts recently created {@link Job} by using {@link JobManager}. */ @Slf4j public class CreateJobTask extends JobTask { diff --git a/core/src/main/java/feast/core/job/task/JobTask.java b/job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java similarity index 94% rename from core/src/main/java/feast/core/job/task/JobTask.java rename to job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java index 60a51809d67..0fa2b72cdda 100644 --- a/core/src/main/java/feast/core/job/task/JobTask.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.core.job.JobManager; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import java.util.concurrent.Callable; import lombok.Getter; import lombok.Setter; diff --git a/core/src/main/java/feast/core/job/task/JobTasks.java b/job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java similarity index 96% rename from core/src/main/java/feast/core/job/task/JobTasks.java rename to job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java index 85b52bdfdeb..bb0148cd8cf 100644 --- a/core/src/main/java/feast/core/job/task/JobTasks.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; /** Enum listing of the available Job Tasks to perform on Jobs */ public enum JobTasks { diff --git a/core/src/main/java/feast/core/job/task/RestartJobTask.java b/job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java similarity index 92% rename from core/src/main/java/feast/core/job/task/RestartJobTask.java rename to job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java index 2c226c3ecc3..1f9ebac2a4a 100644 --- a/core/src/main/java/feast/core/job/task/RestartJobTask.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.core.job.JobManager; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; diff --git a/core/src/main/java/feast/core/job/task/TerminateJobTask.java b/job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java similarity index 92% rename from core/src/main/java/feast/core/job/task/TerminateJobTask.java rename to job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java index 5099d2d2049..b5df9f5d271 100644 --- a/core/src/main/java/feast/core/job/task/TerminateJobTask.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.core.job.JobManager; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; diff --git a/core/src/main/java/feast/core/job/task/UpdateJobStatusTask.java b/job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java similarity index 90% rename from core/src/main/java/feast/core/job/task/UpdateJobStatusTask.java rename to job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java index c793ab6a815..fd27757b13b 100644 --- a/core/src/main/java/feast/core/job/task/UpdateJobStatusTask.java +++ b/job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; -import feast.core.job.JobManager; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; /** * Task that retrieves status from {@link JobManager} on given {@link Job} and update the job diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java similarity index 88% rename from core/src/main/java/feast/core/service/JobCoordinatorService.java rename to job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java index c2e17048fcc..6d64c1c9ffd 100644 --- a/core/src/main/java/feast/core/service/JobCoordinatorService.java +++ b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java @@ -14,29 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.service; +package feast.jc.service; import static feast.common.models.Store.isSubscribedToFeatureSet; -import static feast.core.model.FeatureSet.parseReference; import com.google.common.collect.Sets; -import com.google.protobuf.InvalidProtocolBufferException; import feast.common.models.FeatureSetReference; -import feast.core.config.FeastProperties; -import feast.core.config.FeastProperties.JobProperties; -import feast.core.job.*; -import feast.core.job.task.*; -import feast.core.model.FeatureSetDeliveryStatus; -import feast.core.model.Job; -import feast.core.model.JobStatus; -import feast.proto.core.CoreServiceProto; +import feast.jc.config.FeastProperties; +import feast.jc.config.FeastProperties.JobProperties; +import feast.jc.dao.JobRepository; +import feast.jc.model.FeatureSetDeliveryStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobGroupingStrategy; +import feast.jc.runner.JobManager; +import feast.jc.runner.task.CreateJobTask; +import feast.jc.runner.task.JobTask; +import feast.jc.runner.task.TerminateJobTask; +import feast.jc.runner.task.UpdateJobStatusTask; +import feast.proto.core.*; import feast.proto.core.CoreServiceProto.ListStoresRequest.Filter; import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.FeatureSetProto; import feast.proto.core.FeatureSetProto.FeatureSet; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.core.IngestionJobProto; import feast.proto.core.SourceProto.Source; import feast.proto.core.StoreProto.Store; import java.util.*; @@ -62,7 +62,7 @@ public class JobCoordinatorService { public static final String VERSION_LABEL = "feast_version"; private final JobRepository jobRepository; - private final SpecService specService; + private final CoreServiceGrpc.CoreServiceBlockingStub specService; private final JobManager jobManager; private final JobProperties jobProperties; private final JobGroupingStrategy groupingStrategy; @@ -75,7 +75,7 @@ public class JobCoordinatorService { @Autowired public JobCoordinatorService( JobRepository jobRepository, - SpecService specService, + CoreServiceGrpc.CoreServiceBlockingStub specService, JobManager jobManager, FeastProperties feastProperties, JobGroupingStrategy groupingStrategy, @@ -309,7 +309,11 @@ private Collection getExtraJobs(List keepJobs) { } private List getAllStores() { - ListStoresResponse listStoresResponse = specService.listStores(Filter.newBuilder().build()); + ListStoresResponse listStoresResponse = + specService.listStores( + CoreServiceProto.ListStoresRequest.newBuilder() + .setFilter(Filter.newBuilder().build()) + .build()); return listStoresResponse.getStoreList().stream() .filter(s -> this.whitelistedStores.contains(s.getName())) .collect(Collectors.toList()); @@ -345,41 +349,40 @@ List getFeatureSetsForStore(Store store) { return store.getSubscriptionsList().stream() .flatMap( subscription -> { - try { - return specService - .listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(subscription.getProject()) - .setFeatureSetName(subscription.getName()) - .build()) - .getFeatureSetsList().stream() - .filter( - f -> - this.featureSetSubscriptions.isEmpty() - || isSubscribedToFeatureSet( - this.featureSetSubscriptions, - f.getSpec().getProject(), - f.getSpec().getName())); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - String.format( - "Couldn't fetch featureSets for subscription %s. Reason: %s", - subscription, e.getMessage())); - } + return specService + .listFeatureSets( + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() + .setProject(subscription.getProject()) + .setFeatureSetName(subscription.getName()) + .build()) + .build()) + .getFeatureSetsList().stream() + .filter( + f -> + this.featureSetSubscriptions.isEmpty() + || isSubscribedToFeatureSet( + this.featureSetSubscriptions, + f.getSpec().getProject(), + f.getSpec().getName())); }) .distinct() .collect(Collectors.toList()); } @Scheduled(fixedDelayString = "${feast.stream.specsOptions.notifyIntervalMilliseconds}") - public void notifyJobsWhenFeatureSetUpdated() throws InvalidProtocolBufferException { + public void notifyJobsWhenFeatureSetUpdated() { List pendingFeatureSets = specService .listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject("*") - .setFeatureSetName("*") - .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_PENDING) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() + .setProject("*") + .setFeatureSetName("*") + .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_PENDING) + .build()) .build()) .getFeatureSetsList(); @@ -460,16 +463,16 @@ public void notifyJobsWhenFeatureSetUpdated() throws InvalidProtocolBufferExcept @KafkaListener( topics = {"${feast.stream.specsOptions.specsAckTopic}"}, containerFactory = "kafkaAckListenerContainerFactory") - public void listenAckFromJobs(ConsumerRecord record) - throws InvalidProtocolBufferException { + public void listenAckFromJobs( + ConsumerRecord record) { String setReference = record.key(); - Pair projectAndSetName = parseReference(setReference); + FeatureSetReference ref = FeatureSetReference.parse(setReference); FeatureSet featureSet = specService .getFeatureSet( CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setProject(projectAndSetName.getLeft()) - .setName(projectAndSetName.getRight()) + .setProject(ref.getProjectName()) + .setName(ref.getFeatureSetName()) .build()) .getFeatureSet(); @@ -489,9 +492,6 @@ public void listenAckFromJobs(ConsumerRecord convertTagStringToList(String tags) { + if (tags == null || tags.isEmpty()) { + return Collections.emptyList(); + } + return Arrays.asList(tags.split(",")); + } + + /** + * Unmarshals a given json string to map + * + * @param jsonString valid json formatted string + * @return map of keys to values in json + */ + public static Map convertJsonStringToMap(String jsonString) { + if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { + return Collections.emptyMap(); + } + Type stringMapType = new TypeToken>() {}.getType(); + return gson.fromJson(jsonString, stringMapType); + } + + /** + * Marshals a given map into its corresponding json string + * + * @param map + * @return json string corresponding to given map + */ + public static String convertMapToJsonString(Map map) { + return gson.toJson(map); + } +} diff --git a/job-coordinator/src/main/resources/application.yml b/job-coordinator/src/main/resources/application.yml new file mode 100644 index 00000000000..8d05afe4073 --- /dev/null +++ b/job-coordinator/src/main/resources/application.yml @@ -0,0 +1,133 @@ +# +# Copyright 2018 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +feast: + core-host: localhost + core-port: 6565 + + jobs: + # Enabling JobManagement + enabled: true + + # Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner. + polling_interval_milliseconds: 60000 + + # Timeout in seconds for each attempt to update or submit a new job to the runner. + job_update_timeout_seconds: 240 + + # Name of the active runner in "runners" that should be used. Only a single runner can be active at one time. + active_runner: direct + + # List of runner configurations. Please see protos/feast/core/Runner.proto for more details + # Alternatively see the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner + runners: + - name: direct + type: DirectRunner + options: + tempLocation: gs://bucket/tempLocation + + - name: dataflow + type: DataflowRunner + options: + project: my_gcp_project + region: asia-east1 + workerZone: asia-east1-a + tempLocation: gs://bucket/tempLocation + network: default + subnetwork: regions/asia-east1/subnetworks/mysubnetwork + maxNumWorkers: 1 + enableStreamingEngine: false + workerDiskType: compute.googleapis.com/projects/asia-east1-a/diskTypes/pd-ssd + autoscalingAlgorithm: THROUGHPUT_BASED + usePublicIps: false + workerMachineType: n1-standard-1 + deadLetterTableSpec: project_id:dataset_id.table_id + + # Configuration options for metric collection for all ingestion jobs + metrics: + # Enable metrics pushing for all ingestion jobs. + enabled: false + # Type of metrics sink. Only statsd is currently supported. + type: statsd + # Host of the metrics sink. + host: localhost + # Port of the metrics sink. + port: 9125 + + coordinator: + # if true one job per source with many stores would be created + # if false one job per source-store pair would be created + consolidate-jobs-per-source: false + + # labels (map) that being assigned to job on creation. + # And also used to determine jobs that are being managed by current application + # among all running jobs + jobSelector: + application: feast + + # Specify feature sets that should be handled by current instance of JobManager + featureSetSelector: + - project: "*" + name: "*" + # Stores names that are enabled on current instance of JobManager + whitelisted-stores: + - online + - online_cluster + - historical + + stream: + # Feature stream type. Only kafka is supported. + type: kafka + # Feature stream options. + # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig + options: + topic: feast-features + bootstrapServers: localhost:9092 + replicationFactor: 1 + partitions: 1 + specsOptions: + specsTopic: feast-specs + specsAckTopic: feast-specs-ack + notifyIntervalMilliseconds: 1000 + + logging: + # Audit logging provides a machine readable structured JSON log that can give better + # insight into what is happening in Feast. + audit: + # Whether audit logging is enabled. + enabled: true + # Whether to enable message level (ie request/response) audit logging + messageLoggingEnabled: false + +grpc: + server: + # The port that Feast Core gRPC service listens on + port: 6567 + security: + enabled: false + certificateChain: server.crt + privateKey: server.key + +management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: true + host: ${STATSD_HOST:localhost} + port: ${STATSD_PORT:8125} diff --git a/job-coordinator/src/main/resources/banner.txt b/job-coordinator/src/main/resources/banner.txt new file mode 100644 index 00000000000..c2bc81e3e0d --- /dev/null +++ b/job-coordinator/src/main/resources/banner.txt @@ -0,0 +1,21 @@ + +███████╗███████╗ █████╗ ███████╗████████╗ +██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝ +█████╗ █████╗ ███████║███████╗ ██║ +██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║ +██║ ███████╗██║ ██║███████║ ██║ +╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ + + ██╗ ██████╗ ██████╗ + ██║██╔═══██╗██╔══██╗ + ██║██║ ██║██████╔╝ +██ ██║██║ ██║██╔══██╗ +╚█████╔╝╚██████╔╝██████╔╝ + ╚════╝ ╚═════╝ ╚═════╝ + + ██████╗ ██████╗ ██████╗ ██████╗ ██████╗ ██╗███╗ ██╗ █████╗ ████████╗ ██████╗ ██████╗ +██╔════╝██╔═══██╗██╔═══██╗██╔══██╗██╔══██╗██║████╗ ██║██╔══██╗╚══██╔══╝██╔═══██╗██╔══██╗ +██║ ██║ ██║██║ ██║██████╔╝██║ ██║██║██╔██╗ ██║███████║ ██║ ██║ ██║██████╔╝ +██║ ██║ ██║██║ ██║██╔══██╗██║ ██║██║██║╚██╗██║██╔══██║ ██║ ██║ ██║██╔══██╗ +╚██████╗╚██████╔╝╚██████╔╝██║ ██║██████╔╝██║██║ ╚████║██║ ██║ ██║ ╚██████╔╝██║ ██║ + ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚═════╝ ╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ diff --git a/core/src/test/java/feast/core/model/JobStatusTest.java b/job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java similarity index 92% rename from core/src/test/java/feast/core/model/JobStatusTest.java rename to job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java index f5c8839386c..90ea68a6c05 100644 --- a/core/src/test/java/feast/core/model/JobStatusTest.java +++ b/job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.model; +package feast.jc.model; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class JobStatusTest { diff --git a/core/src/test/java/feast/core/job/RunnerTest.java b/job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java similarity index 79% rename from core/src/test/java/feast/core/job/RunnerTest.java rename to job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java index ce1700acbe9..d66230e67e8 100644 --- a/core/src/test/java/feast/core/job/RunnerTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job; +package feast.jc.runner; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.NoSuchElementException; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class RunnerTest { @@ -35,8 +36,9 @@ public void fromStringLoadsValueFromHumanReadableName() { assertThat(Runner.fromString(humanName), is(Runner.DATAFLOW)); } - @Test(expected = NoSuchElementException.class) + @Test public void fromStringThrowsNoSuchElementExceptionForUnknownValue() { - Runner.fromString("this is not a valid Runner"); + assertThrows( + NoSuchElementException.class, () -> Runner.fromString("this is not a valid Runner")); } } diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java similarity index 93% rename from core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java rename to job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java index 7b07d52fee3..d422c52a92a 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java @@ -14,24 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.Environment; import com.google.api.services.dataflow.model.ListJobsResponse; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; -import feast.core.config.FeastProperties.MetricsProperties; -import feast.core.exception.JobExecutionException; -import feast.core.model.*; import feast.ingestion.options.ImportOptions; +import feast.jc.config.FeastProperties; +import feast.jc.exception.JobExecutionException; +import feast.jc.model.Job; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions.Builder; @@ -49,17 +51,13 @@ import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.sdk.PipelineResult.State; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; public class DataflowJobManagerTest { - @Rule public final ExpectedException expectedException = ExpectedException.none(); - private Dataflow dataflow; private DataflowRunnerConfigOptions defaults; @@ -69,9 +67,8 @@ public class DataflowJobManagerTest { private StoreProto.Store store; private SourceProto.Source source; - @Before + @BeforeEach public void setUp() { - initMocks(this); Builder optionsBuilder = DataflowRunnerConfigOptions.newBuilder(); optionsBuilder.setProject("project"); optionsBuilder.setRegion("region"); @@ -81,7 +78,7 @@ public void setUp() { optionsBuilder.setSubnetwork("subnetwork"); optionsBuilder.putLabels("orchestrator", "feast"); defaults = optionsBuilder.build(); - MetricsProperties metricsProperties = new MetricsProperties(); + FeastProperties.MetricsProperties metricsProperties = new FeastProperties.MetricsProperties(); metricsProperties.setEnabled(false); dataflow = mock(Dataflow.class, RETURNS_DEEP_STUBS); @@ -208,8 +205,7 @@ public void shouldThrowExceptionWhenJobStateTerminal() throws IOException { .setSource(source) .setStores(ImmutableMap.of(store.getName(), store)) .build(); - expectedException.expect(JobExecutionException.class); - dfJobManager.startJob(job); + assertThrows(JobExecutionException.class, () -> dfJobManager.startJob(job)); } @Test @@ -239,7 +235,7 @@ public void shouldRetrieveRunningJobsFromDataflow() { .execute()) .thenReturn(new com.google.api.services.dataflow.model.Job()); - JsonFormat.Printer jsonPrinter = JsonFormat.printer(); + Printer jsonPrinter = JsonFormat.printer(); when(dataflow .projects() @@ -307,7 +303,7 @@ public void shouldRetrieveRunningJobsWithoutLabels() { ImmutableList.of( new com.google.api.services.dataflow.model.Job().setId("job-1")))); - JsonFormat.Printer jsonPrinter = JsonFormat.printer(); + Printer jsonPrinter = JsonFormat.printer(); // job with no labels when(dataflow @@ -330,7 +326,7 @@ public void shouldRetrieveRunningJobsWithoutLabels() { "sourceJson", jsonPrinter.print(source), "storesJson", ImmutableList.of(jsonPrinter.print(store))))))); - MetricsProperties metricsProperties = new MetricsProperties(); + FeastProperties.MetricsProperties metricsProperties = new FeastProperties.MetricsProperties(); metricsProperties.setEnabled(false); dfJobManager = diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobStateMapperTest.java b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java similarity index 78% rename from core/src/test/java/feast/core/job/dataflow/DataflowJobStateMapperTest.java rename to job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java index af9e8b2c426..ff1783fe0a2 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowJobStateMapperTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java @@ -14,16 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; -import org.junit.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; public class DataflowJobStateMapperTest { private DataflowJobStateMapper mapper = new DataflowJobStateMapper(); - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowIllegalArgumentExceptionForInvalidString() { - mapper.map("INVALID_STATE"); + assertThrows(IllegalArgumentException.class, () -> mapper.map("INVALID_STATE")); } } diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowRunnerConfigTest.java b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java similarity index 97% rename from core/src/test/java/feast/core/job/dataflow/DataflowRunnerConfigTest.java rename to job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java index 9c6b5a085c8..bd49dc82d7a 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowRunnerConfigTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java @@ -14,17 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.dataflow; +package feast.jc.runner.dataflow; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.*; +import static org.junit.Assert.assertThat; import com.google.common.collect.Lists; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import java.util.Arrays; import java.util.List; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class DataflowRunnerConfigTest { @Test diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerConfigTest.java b/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java similarity index 92% rename from core/src/test/java/feast/core/job/direct/DirectRunnerConfigTest.java rename to job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java index c4114dd2104..1678e22c980 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerConfigTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java @@ -14,16 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.*; import com.google.common.collect.Lists; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import java.util.List; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class DirectRunnerConfigTest { @Test diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java similarity index 92% rename from core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java rename to job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java index 269191349a8..6fe573204c6 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java @@ -14,25 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.direct; +package feast.jc.runner.direct; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; -import feast.core.config.FeastProperties.MetricsProperties; -import feast.core.model.Job; -import feast.core.model.JobStatus; import feast.ingestion.options.ImportOptions; +import feast.jc.config.FeastProperties.MetricsProperties; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import feast.proto.core.SourceProto; @@ -46,17 +43,13 @@ import org.apache.beam.runners.direct.DirectRunner; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; public class DirectRunnerJobManagerTest { - @Rule public final ExpectedException expectedException = ExpectedException.none(); - @Mock private DirectJobRegistry directJobRegistry; private DirectRunnerJobManager drJobManager; @@ -66,7 +59,7 @@ public class DirectRunnerJobManagerTest { private StoreProto.Store store; private SourceProto.Source source; - @Before + @BeforeEach public void setUp() { initMocks(this); defaults = DirectRunnerConfigOptions.newBuilder().setTargetParallelism(1).build(); diff --git a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java b/job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java similarity index 96% rename from core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java rename to job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java index a12452b593b..ca2ba5f42c7 100644 --- a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.option; +package feast.jc.runner.option; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.FeatureSetProto; @@ -25,7 +25,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class FeatureSetJsonByteConverterTest { diff --git a/core/src/test/java/feast/core/job/task/JobTasksTest.java b/job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java similarity index 93% rename from core/src/test/java/feast/core/job/task/JobTasksTest.java rename to job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java index 7a0603179fa..729805814aa 100644 --- a/core/src/test/java/feast/core/job/task/JobTasksTest.java +++ b/job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java @@ -14,21 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.job.task; +package feast.jc.runner.task; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; import static org.hamcrest.core.IsEqual.equalTo; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.ImmutableMap; -import feast.core.job.*; -import feast.core.model.Job; -import feast.core.model.JobStatus; -import feast.core.util.TestUtil; +import feast.common.util.TestUtil; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; +import feast.jc.runner.Runner; import feast.proto.core.SourceProto; import feast.proto.core.SourceProto.KafkaSourceConfig; import feast.proto.core.SourceProto.SourceType; @@ -37,8 +36,8 @@ import feast.proto.core.StoreProto.Store.StoreType; import feast.proto.core.StoreProto.Store.Subscription; import lombok.SneakyThrows; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; public class JobTasksTest { @@ -49,7 +48,7 @@ public class JobTasksTest { private StoreProto.Store store; private SourceProto.Source source; - @Before + @BeforeEach public void setUp() { initMocks(this); when(jobManager.getRunnerType()).thenReturn(RUNNER); diff --git a/core/src/test/java/feast/core/service/FakeJobManager.java b/job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java similarity index 92% rename from core/src/test/java/feast/core/service/FakeJobManager.java rename to job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java index 657534eb7b1..27ca84ecaed 100644 --- a/core/src/test/java/feast/core/service/FakeJobManager.java +++ b/job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.service; +package feast.jc.service; import com.google.common.collect.Lists; -import feast.core.job.JobManager; -import feast.core.job.Runner; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; +import feast.jc.runner.Runner; import java.util.*; public class FakeJobManager implements JobManager { diff --git a/core/src/test/java/feast/core/service/JobCoordinatorIT.java b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java similarity index 86% rename from core/src/test/java/feast/core/service/JobCoordinatorIT.java rename to job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java index 7f9233f8325..5a43d0f38a5 100644 --- a/core/src/test/java/feast/core/service/JobCoordinatorIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.service; +package feast.jc.service; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.*; @@ -29,26 +29,40 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.it.SimpleAPIClient; -import feast.core.job.JobManager; -import feast.core.job.JobRepository; -import feast.core.model.*; -import feast.proto.core.*; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; +import feast.common.it.SimpleAPIClient; +import feast.common.util.KafkaSerialization; +import feast.jc.config.FeastProperties; +import feast.jc.dao.JobRepository; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; +import feast.proto.core.CoreServiceGrpc; +import feast.proto.core.FeatureSetProto; +import feast.proto.core.IngestionJobProto; +import feast.proto.core.StoreProto; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.SneakyThrows; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; +import org.springframework.core.env.StandardEnvironment; import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; @SpringBootTest( @@ -76,6 +90,17 @@ public class JobCoordinatorIT extends BaseIT { @BeforeAll public static void globalSetUp(@Value("${grpc.server.port}") int port) { + StandardEnvironment env = new StandardEnvironment(); + env.setDefaultProfiles("it-core"); + new SpringApplicationBuilder(feast.core.CoreApplication.class) + .environment(env) + .properties( + ImmutableMap.of( + "spring.datasource.url", postgreSQLContainer.getJdbcUrl(), + "spring.datasource.username", postgreSQLContainer.getUsername(), + "spring.datasource.password", postgreSQLContainer.getPassword())) + .run(); + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); stub = CoreServiceGrpc.newBlockingStub(channel); @@ -385,5 +410,21 @@ public static class TestConfig extends BaseTestConfig { public JobManager getJobManager() { return new FakeJobManager(); } + + @Bean + public KafkaTemplate specAckKafkaTemplate( + FeastProperties feastProperties) { + FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); + Map props = new HashMap<>(); + + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); + + KafkaTemplate t = + new KafkaTemplate<>( + new DefaultKafkaProducerFactory<>( + props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); + t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsAckTopic()); + return t; + } } } diff --git a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java similarity index 82% rename from core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java rename to job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java index a6f206c0f3e..45895b249c5 100644 --- a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.service; +package feast.jc.service; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -30,15 +30,21 @@ import com.google.common.collect.*; import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.config.FeastProperties; -import feast.core.config.FeastProperties.JobProperties; -import feast.core.it.DataGenerator; -import feast.core.job.*; -import feast.core.job.JobRepository; -import feast.core.job.task.*; -import feast.core.model.Job; -import feast.core.model.JobStatus; -import feast.core.util.TestUtil; +import feast.common.it.DataGenerator; +import feast.jc.config.FeastProperties; +import feast.jc.config.FeastProperties.JobProperties; +import feast.jc.dao.InMemoryJobRepository; +import feast.jc.dao.JobRepository; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.ConsolidatedJobStrategy; +import feast.jc.runner.JobManager; +import feast.jc.runner.JobPerStoreStrategy; +import feast.jc.runner.task.CreateJobTask; +import feast.jc.runner.task.JobTask; +import feast.jc.runner.task.UpdateJobStatusTask; +import feast.proto.core.CoreServiceGrpc; +import feast.proto.core.CoreServiceProto; import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest.Filter; import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; import feast.proto.core.CoreServiceProto.ListStoresResponse; @@ -49,39 +55,35 @@ import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.springframework.kafka.core.KafkaTemplate; public class JobCoordinatorServiceTest { - @Rule public final ExpectedException exception = ExpectedException.none(); - JobRepository jobRepository; - @Mock SpecService specService; + @Mock CoreServiceGrpc.CoreServiceBlockingStub specService; private FeastProperties feastProperties; private JobCoordinatorService jcsWithConsolidation; private JobCoordinatorService jcsWithJobPerStore; - @Before + @BeforeEach public void setUp() { initMocks(this); feastProperties = new FeastProperties(); JobProperties jobProperties = new JobProperties(); jobProperties.setJobUpdateTimeoutSeconds(5); - JobProperties.CoordinatorProperties.FeatureSetSelector selector = - new JobProperties.CoordinatorProperties.FeatureSetSelector(); + FeastProperties.JobProperties.CoordinatorProperties.FeatureSetSelector selector = + new FeastProperties.JobProperties.CoordinatorProperties.FeatureSetSelector(); selector.setName("fs*"); selector.setProject("*"); - JobProperties.CoordinatorProperties coordinatorProperties = - new JobProperties.CoordinatorProperties(); + FeastProperties.JobProperties.CoordinatorProperties coordinatorProperties = + new FeastProperties.JobProperties.CoordinatorProperties(); coordinatorProperties.setFeatureSetSelector(ImmutableList.of(selector)); coordinatorProperties.setWhitelistedStores( ImmutableList.of("test-store", "test", "test-1", "test-2", "normal-store")); @@ -91,8 +93,6 @@ public void setUp() { feastProperties.setJobs(jobProperties); feastProperties.setVersion("1.0.0"); - TestUtil.setupAuditLogger(); - JobManager jobManager = mock(JobManager.class); when(jobManager.listRunningJobs()).thenReturn(Collections.emptyList()); @@ -134,7 +134,9 @@ public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocol when(specService.listStores(any())) .thenReturn(ListStoresResponse.newBuilder().addStore(storeSpec).build()); when(specService.listFeatureSets( - Filter.newBuilder().setProject("*").setFeatureSetName("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter(Filter.newBuilder().setProject("*").setFeatureSetName("*").build()) + .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List jobTasks = @@ -157,7 +159,10 @@ public void shouldGroupJobsBySource() { FeatureSet featureSet2 = DataGenerator.createFeatureSet(source2, "project1", "fs2"); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("*").setProject("project1").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + Filter.newBuilder().setFeatureSetName("*").setProject("project1").build()) + .build())) .thenReturn( ListFeatureSetsResponse.newBuilder() .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2)) @@ -191,14 +196,20 @@ public void shouldUseStoreSubscriptionToMapStore() { FeatureSet featureSet2 = DataGenerator.createFeatureSet(source2, "default", "fs2"); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("features1").setProject("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + Filter.newBuilder().setFeatureSetName("features1").setProject("*").build()) + .build())) .thenReturn( ListFeatureSetsResponse.newBuilder() .addAllFeatureSets(Lists.newArrayList(featureSet1)) .build()); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("features2").setProject("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + Filter.newBuilder().setFeatureSetName("features2").setProject("*").build()) + .build())) .thenReturn( ListFeatureSetsResponse.newBuilder() .addAllFeatureSets(Lists.newArrayList(featureSet2)) @@ -264,7 +275,9 @@ public void shouldCreateJobIfNoRunning() { Store store = DataGenerator.getDefaultStore(); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("*").setProject("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) + .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List tasks = @@ -288,7 +301,9 @@ public void shouldCreateJobPerStore() throws InvalidProtocolBufferException { FeatureSet featureSet = DataGenerator.createFeatureSet(source, "default", "fs1"); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("*").setProject("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) + .build())) .thenReturn( ListFeatureSetsResponse.newBuilder() .addAllFeatureSets(Lists.newArrayList(featureSet)) @@ -361,7 +376,9 @@ public void shouldSelectOnlyFeatureSetsThatJobManagerSubscribedTo() { FeatureSet featureSet3 = DataGenerator.createFeatureSet(source, "default", "not-fs"); when(specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("*").setProject("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) + .build())) .thenReturn( ListFeatureSetsResponse.newBuilder() .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2, featureSet3)) @@ -373,7 +390,7 @@ public void shouldSelectOnlyFeatureSetsThatJobManagerSubscribedTo() { @Test @SneakyThrows - public void shouldSelectOnlyStoresThatNotBlacklisted() { + public void shouldSelectOnlyStoresThatWhitelisted() { Store store1 = DataGenerator.createStore( "normal-store", @@ -395,11 +412,17 @@ public void shouldSelectOnlyStoresThatNotBlacklisted() { .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); when(specService.listFeatureSets( - Filter.newBuilder().setProject("project1").setFeatureSetName("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + Filter.newBuilder().setProject("project1").setFeatureSetName("*").build()) + .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet1).build()); when(specService.listFeatureSets( - Filter.newBuilder().setProject("project2").setFeatureSetName("*").build())) + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + Filter.newBuilder().setProject("project2").setFeatureSetName("*").build()) + .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet2).build()); ArrayList>> pairs = diff --git a/core/src/test/java/feast/core/service/JobServiceIT.java b/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java similarity index 95% rename from core/src/test/java/feast/core/service/JobServiceIT.java rename to job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java index ec17546f092..ed20cb0cac3 100644 --- a/core/src/test/java/feast/core/service/JobServiceIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.core.service; +package feast.jc.service; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -23,14 +23,14 @@ import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import com.google.common.collect.ImmutableMap; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; import feast.common.models.FeatureSetReference; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.job.JobManager; -import feast.core.job.JobRepository; -import feast.core.model.FeatureSetDeliveryStatus; -import feast.core.model.Job; -import feast.core.model.JobStatus; +import feast.jc.dao.JobRepository; +import feast.jc.model.FeatureSetDeliveryStatus; +import feast.jc.model.Job; +import feast.jc.model.JobStatus; +import feast.jc.runner.JobManager; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceProto; import feast.proto.core.FeatureSetReferenceProto; @@ -155,7 +155,7 @@ public void shouldReturnListOfJobsByByFeatureSetReference() { } @Test - public void shoulStopJobById() { + public void shouldStopJobById() { CoreServiceProto.StopIngestionJobRequest request = CoreServiceProto.StopIngestionJobRequest.newBuilder().setId(this.job.getId()).build(); diff --git a/job-coordinator/src/test/resources/application-it-core.yml b/job-coordinator/src/test/resources/application-it-core.yml new file mode 100644 index 00000000000..9ad4e7037cd --- /dev/null +++ b/job-coordinator/src/test/resources/application-it-core.yml @@ -0,0 +1,67 @@ +feast: + stream: + # Feature stream type. Only kafka is supported. + type: kafka + # Feature stream options. + # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig + options: + topic: feast-features + bootstrapServers: localhost:9092 + replicationFactor: 1 + partitions: 1 + specsOptions: + specsTopic: feast-specs + specsAckTopic: feast-specs-ack + notifyIntervalMilliseconds: 1000 + + security: + authentication: + enabled: false + provider: jwt + options: + jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" + + authorization: + enabled: false + provider: http + options: + authorizationUrl: http://localhost:8082 + subjectClaim: email + + logging: + # Audit logging provides a machine readable structured JSON log that can give better + # insight into what is happening in Feast. + audit: + # Whether audit logging is enabled. + enabled: true + # Whether to enable message level (ie request/response) audit logging + messageLoggingEnabled: false + +grpc: + server: + # The port that Feast Core gRPC service listens on + port: 6565 + security: + enabled: false + +spring: + jpa: + properties.hibernate: + format_sql: true + event: + merge: + entity_copy_observer: allow + hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl + hibernate.ddl-auto: none + datasource: + driverClassName: org.postgresql.Driver + +management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: true + host: ${STATSD_HOST:localhost} + port: ${STATSD_PORT:8125} diff --git a/job-coordinator/src/test/resources/application-it.properties b/job-coordinator/src/test/resources/application-it.properties new file mode 100644 index 00000000000..ccb339fa9ab --- /dev/null +++ b/job-coordinator/src/test/resources/application-it.properties @@ -0,0 +1,31 @@ +# +# Copyright 2018 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +grpc.server.port=6666 + +feast.security.authentication.enabled = false +feast.security.authorization.enabled = false + +feast.jobs.enabled=false + +spring.jpa.properties.hibernate.format_sql=true +spring.jpa.properties.hibernate.show_sql=false +spring.jpa.hibernate.naming.physical-strategy=org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy +spring.jpa.hibernate.ddl-auto=none + +spring.datasource.hikari.maximum-pool-size=100 +spring.main.allow-bean-definition-overriding=true + diff --git a/job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..ca6ee9cea8e --- /dev/null +++ b/job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5563ef6d5a4..111acb6c3b3 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,8 @@ docs/coverage/java common auth + job-coordinator + common-test From 15ea27ce417c9bcdc9b48398f41804e4b4a3b8db Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 17 Aug 2020 17:56:42 +0800 Subject: [PATCH 02/26] unify baseIT --- common-test/pom.xml | 23 ++- .../src/main/java/feast/common/it/BaseIT.java | 51 +++---- common/pom.xml | 4 - core/pom.xml | 134 +++--------------- .../auth/CoreServiceAuthenticationIT.java | 10 +- .../test/resources/application-it.properties | 2 +- ingestion/pom.xml | 6 - job-coordinator/pom.xml | 115 ++------------- .../feast/jc/service/JobCoordinatorIT.java | 31 ++-- pom.xml | 13 ++ serving/pom.xml | 6 - 11 files changed, 107 insertions(+), 288 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 8f7efb8c79f..3a2d44d5eb0 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -92,12 +92,16 @@ org.springframework.boot spring-boot-test - 2.3.1.RELEASE + ${spring.boot.version} + + + org.springframework.boot + spring-boot-test-autoconfigure org.springframework spring-test - 5.2.5.RELEASE + ${spring.version} org.testcontainers @@ -149,5 +153,20 @@ ${mockito.version} compile + + io.rest-assured + rest-assured + 4.2.0 + + + io.rest-assured + json-path + 4.2.0 + + + io.rest-assured + xml-path + 4.2.0 + diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java index 529dc85905c..0a98043860a 100644 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -17,20 +17,14 @@ package feast.common.it; import io.prometheus.client.CollectorRegistry; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Table; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.hibernate.engine.spi.SessionImplementor; import org.junit.jupiter.api.*; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; @@ -89,7 +83,7 @@ static void properties(DynamicPropertyRegistry registry) { public class SequentialFlow { @AfterAll public void tearDown() throws Exception { - cleanTables(entityManager); + cleanTables(); } } @@ -123,26 +117,23 @@ public ConsumerFactory testConsumerFactory() { /** * Truncates all tables in Database (between tests or flows). Retries on deadlock * - * @param em EntityManager * @throws SQLException */ - public static void cleanTables(EntityManager em) throws SQLException { - List tableNames = - em.getMetamodel().getEntities().stream() - .map(e -> e.getJavaType().getAnnotation(Table.class).name()) - .collect(Collectors.toList()); - - // this trick needed to get EntityManager with Transaction - // and we don't want to wrap whole class into @Transactional - em = em.getEntityManagerFactory().createEntityManager(); - // Transaction needed only once to do unwrap - SessionImplementor session = em.unwrap(SessionImplementor.class); - - // and here we're actually don't want any transactions - // but instead we pulling raw connection - // to be able to retry query if needed - // since retrying rollbacked transaction is not that easy - Connection connection = session.connection(); + public static void cleanTables() throws SQLException { + Connection connection = + DriverManager.getConnection( + postgreSQLContainer.getJdbcUrl(), + postgreSQLContainer.getUsername(), + postgreSQLContainer.getPassword()); + + List tableNames = new ArrayList<>(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema='public'"); + while (rs.next()) { + tableNames.add(rs.getString(1)); + } // retries are needed since truncate require exclusive lock // and that often leads to Deadlock @@ -150,7 +141,7 @@ public static void cleanTables(EntityManager em) throws SQLException { int num_retries = 5; for (int i = 1; i <= num_retries; i++) { try { - Statement statement = connection.createStatement(); + statement = connection.createStatement(); statement.execute(String.format("truncate %s cascade", String.join(", ", tableNames))); } catch (SQLException e) { if (i == num_retries) { @@ -163,8 +154,6 @@ public static void cleanTables(EntityManager em) throws SQLException { } } - @PersistenceContext EntityManager entityManager; - /** Used to determine SequentialFlows */ public Boolean isSequentialTest(TestInfo testInfo) { try { @@ -180,7 +169,7 @@ public void tearDown(TestInfo testInfo) throws Exception { CollectorRegistry.defaultRegistry.clear(); if (!isSequentialTest(testInfo)) { - cleanTables(entityManager); + cleanTables(); } } } diff --git a/common/pom.xml b/common/pom.xml index efff48bd04e..71d64f5f4c3 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -68,10 +68,6 @@ com.google.auto.value auto-value-annotations - - com.google.auto.value - auto-value - com.google.code.gson gson diff --git a/core/pom.xml b/core/pom.xml index 80e201472ca..aaf24e2cc49 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -69,18 +69,6 @@ - - dev.feast - feast-ingestion - ${project.version} - - - - org.slf4j - slf4j-simple - - - dev.feast feast-common @@ -97,6 +85,12 @@ ${project.version} test + + dev.feast + feast-storage-connector-bigquery + ${project.version} + compile + @@ -252,11 +246,6 @@ ${lombok.version} - - org.hamcrest - hamcrest-library - - io.prometheus simpleclient @@ -283,24 +272,24 @@ 3.10.0 - - com.jayway.jsonpath - json-path-assert - 2.2.0 - test + org.apache.commons + commons-lang3 - org.springframework.boot - spring-boot-test - test + joda-time + joda-time + + - org.springframework.boot - spring-boot-test-autoconfigure + com.jayway.jsonpath + json-path-assert + 2.2.0 test + javax.xml.bind jaxb-api @@ -315,40 +304,11 @@ hibernate-validator-annotation-processor 6.1.2.Final - - org.mockito - mockito-core - ${mockito.version} - test - - - org.springframework - spring-test - 5.2.5.RELEASE - test - - - org.junit.jupiter - junit-jupiter-api - 5.6.2 - test - - - org.junit.jupiter - junit-jupiter-params - 5.6.2 - test - - - org.testcontainers - testcontainers - 1.14.3 - test - + - org.testcontainers - junit-jupiter - 1.14.3 + dev.feast + feast-common-test + ${project.version} test @@ -357,65 +317,11 @@ 0.4.4-alpha.1 test - - org.testcontainers - postgresql - 1.14.3 - test - - - org.testcontainers - kafka - 1.14.3 - test - com.github.tomakehurst wiremock 2.27.0 test - - org.awaitility - awaitility - 3.0.0 - test - - - org.awaitility - awaitility-proxy - 3.0.0 - test - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - com.google.auto.value - auto-value - 1.6.6 - provided - - - io.rest-assured - rest-assured - 4.2.0 - test - - - io.rest-assured - json-path - 4.2.0 - test - - - io.rest-assured - xml-path - 4.2.0 - test - diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index 78995aa8061..9f62ea98d90 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -40,6 +40,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; @@ -52,6 +53,8 @@ public class CoreServiceAuthenticationIT extends BaseIT { @Autowired FeastProperties feastProperties; + @Autowired Map m; + private static int feast_core_port; private static int JWKS_PORT = 45124; @@ -176,7 +179,12 @@ void canApplyFeatureSetIfAuthenticated() { } @TestConfiguration - public static class TestConfig extends BaseTestConfig {} + public static class TestConfig extends BaseTestConfig { + @Bean + public Map testMap() { + return Collections.emptyMap(); + } + } // Create secure Feast Core gRPC client for a specific user private static SimpleAPIClient getSecureApiClient(String subjectEmail) { diff --git a/core/src/test/resources/application-it.properties b/core/src/test/resources/application-it.properties index 80d1baea473..81aed45a7a9 100644 --- a/core/src/test/resources/application-it.properties +++ b/core/src/test/resources/application-it.properties @@ -21,6 +21,6 @@ feast.security.authorization.enabled = false feast.jobs.enabled=false -spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.maximum-pool-size=50 spring.main.allow-bean-definition-overriding=true diff --git a/ingestion/pom.xml b/ingestion/pom.xml index b358e2f57a9..871dcb46e61 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -149,12 +149,6 @@ auto-value-annotations 1.6.6 - - com.google.auto.value - auto-value - 1.6.6 - provided - com.google.cloud diff --git a/job-coordinator/pom.xml b/job-coordinator/pom.xml index f45d26dd9c6..e70394bb472 100644 --- a/job-coordinator/pom.xml +++ b/job-coordinator/pom.xml @@ -92,6 +92,12 @@ ${project.version} test + + dev.feast + feast-common-test + ${project.version} + test + @@ -120,61 +126,11 @@ org.apache.logging.log4j log4j-web - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security.oauth - spring-security-oauth2 - ${spring.security.oauth2.version} - - - org.springframework.security - spring-security-oauth2-client - ${spring.security.version} - - - org.springframework.security - spring-security-web - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-resource-server - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - net.devh grpc-server-spring-boot-starter ${grpc.spring.boot.starter.version} - - com.nimbusds - nimbus-jose-jwt - 8.2.1 - - - org.springframework.security - spring-security-oauth2-core - ${spring.security.version} - - - - org.springframework.boot - spring-boot-starter-data-jpa - org.springframework.boot @@ -235,11 +191,6 @@ ${lombok.version} - - org.hamcrest - hamcrest-library - - io.prometheus simpleclient @@ -254,17 +205,6 @@ google-api-client-googleapis-auth-oauth 1.2.3-alpha - - com.auth0 - jwks-rsa - 0.11.0 - - - - com.auth0 - java-jwt - 3.10.0 - @@ -274,56 +214,17 @@ test - - org.springframework.boot - spring-boot-test - test - - - org.springframework.boot - spring-boot-test-autoconfigure - test - javax.xml.bind jaxb-api - - org.mockito - mockito-core - ${mockito.version} - test - - - dev.feast - feast-common-test - ${project.version} - test - - - sh.ory.keto - keto-client - 0.4.4-alpha.1 - test - - - com.github.tomakehurst - wiremock - 2.27.0 - test - + + com.google.auto.value auto-value-annotations 1.6.6 - - com.google.auto.value - auto-value - 1.6.6 - provided - - org.postgresql diff --git a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java index 5a43d0f38a5..26fbdaf3f79 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java @@ -56,11 +56,9 @@ import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; -import org.springframework.core.env.StandardEnvironment; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; @@ -68,7 +66,7 @@ @SpringBootTest( properties = { "feast.jobs.enabled=true", - "feast.jobs.polling_interval_milliseconds=1000", + "feast.jobs.polling_interval_milliseconds=10000", "feast.stream.specsOptions.notifyIntervalMilliseconds=100", "feast.jobs.coordinator.consolidate-jobs-per-source=true", "feast.jobs.coordinator.feature-set-selector[0].name=test", @@ -78,7 +76,8 @@ "feast.version=1.0.0" }) public class JobCoordinatorIT extends BaseIT { - @Autowired private FakeJobManager jobManager; + @Autowired private JobManager jobManager2; + private FakeJobManager jobManager; @Autowired private JobRepository jobRepository; @@ -90,16 +89,16 @@ public class JobCoordinatorIT extends BaseIT { @BeforeAll public static void globalSetUp(@Value("${grpc.server.port}") int port) { - StandardEnvironment env = new StandardEnvironment(); - env.setDefaultProfiles("it-core"); - new SpringApplicationBuilder(feast.core.CoreApplication.class) - .environment(env) - .properties( - ImmutableMap.of( - "spring.datasource.url", postgreSQLContainer.getJdbcUrl(), - "spring.datasource.username", postgreSQLContainer.getUsername(), - "spring.datasource.password", postgreSQLContainer.getPassword())) - .run(); + // StandardEnvironment env = new StandardEnvironment(); + // env.setDefaultProfiles("it-core"); + // new SpringApplicationBuilder(feast.core.CoreApplication.class) + // .environment(env) + // .properties( + // ImmutableMap.of( + // "spring.datasource.url", postgreSQLContainer.getJdbcUrl(), + // "spring.datasource.username", postgreSQLContainer.getUsername(), + // "spring.datasource.password", postgreSQLContainer.getPassword())) + // .run(); ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); @@ -386,7 +385,7 @@ public void shouldReallocateFeatureSetAfterSourceChanged() { @Test @Order(6) - public void shouldUpdateStatusAfterACKfromNewJob() { + public void shouldUpdateStatusAfterACKFromNewJob() { job = jobRepository.findByStatus(JobStatus.RUNNING).get(0); ackPublisher.sendDefault( @@ -405,7 +404,7 @@ public void shouldUpdateStatusAfterACKfromNewJob() { } @TestConfiguration - public static class TestConfig extends BaseTestConfig { + public static class TestConfig { @Bean public JobManager getJobManager() { return new FakeJobManager(); diff --git a/pom.xml b/pom.xml index 111acb6c3b3..8bb56a9fe55 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ 3.12.2 3.12.2 2.3.1.RELEASE + 5.2.5.RELEASE 5.3.0.RELEASE 2.9.0.RELEASE 2.22.0 @@ -491,6 +492,18 @@ 8 + + + com.google.auto.value + auto-value + ${auto.value.version} + + + org.projectlombok + lombok + ${lombok.version} + + diff --git a/serving/pom.xml b/serving/pom.xml index e397b11ede4..d0edf6ade0f 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -223,12 +223,6 @@ auto-value-annotations 1.6.6 - - com.google.auto.value - auto-value - 1.6.6 - provided - From 0632fdfe5157ce749b9cd932642f7d031ce7a0b8 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 11:12:40 +0800 Subject: [PATCH 03/26] move ingestion job related code to contrib --- .../java/feast/common/it/ExternalApp.java | 2 + .../java/feast/jc/grpc/CoreServiceImpl.java | 125 ++++++++++++++++ protos/feast/core/CoreService.proto | 18 ++- sdk/python/feast/cli.py | 7 +- sdk/python/feast/client.py | 79 +--------- sdk/python/feast/constants.py | 3 + sdk/python/feast/contrib/__init__.py | 0 .../feast/contrib/job_coordinator/__init__.py | 0 .../feast/contrib/job_coordinator/client.py | 140 ++++++++++++++++++ .../feast/contrib/job_coordinator/job.py | 121 +++++++++++++++ sdk/python/feast/job.py | 117 --------------- sdk/python/tests/test_client.py | 61 ++++---- storage/connectors/bigquery/pom.xml | 7 - .../bigquery/writer/BigQueryWrite.java | 18 +-- 14 files changed, 441 insertions(+), 257 deletions(-) create mode 100644 common-test/src/main/java/feast/common/it/ExternalApp.java create mode 100644 job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java create mode 100644 sdk/python/feast/contrib/__init__.py create mode 100644 sdk/python/feast/contrib/job_coordinator/__init__.py create mode 100644 sdk/python/feast/contrib/job_coordinator/client.py create mode 100644 sdk/python/feast/contrib/job_coordinator/job.py diff --git a/common-test/src/main/java/feast/common/it/ExternalApp.java b/common-test/src/main/java/feast/common/it/ExternalApp.java new file mode 100644 index 00000000000..cd8a3c44edf --- /dev/null +++ b/common-test/src/main/java/feast/common/it/ExternalApp.java @@ -0,0 +1,2 @@ +package feast.common.it;public class ExternalApp { +} diff --git a/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java b/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java new file mode 100644 index 00000000000..9c71d396008 --- /dev/null +++ b/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.jc.grpc; + +import com.google.api.gax.rpc.InvalidArgumentException; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.auth.service.AuthorizationService; +import feast.common.interceptors.GrpcMessageInterceptor; +import feast.jc.config.FeastProperties; +import feast.jc.service.JobService; +import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase; +import feast.proto.core.CoreServiceProto.*; +import feast.proto.core.FeatureSetProto.FeatureSet; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.server.service.GrpcService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +/** Implementation of the feast core GRPC service. */ +@Slf4j +@GrpcService(interceptors = {GrpcMessageInterceptor.class}) +public class CoreServiceImpl extends CoreServiceImplBase { + + private final FeastProperties feastProperties; + + private JobService jobService; + + @Autowired + public CoreServiceImpl( + JobService jobService, + FeastProperties feastProperties) { + this.jobService = jobService; + this.feastProperties = feastProperties; + } + + @Override + public void listIngestionJobs( + ListIngestionJobsRequest request, + StreamObserver responseObserver) { + try { + ListIngestionJobsResponse response = this.jobService.listJobs(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (InvalidArgumentException e) { + log.error("Received an invalid request on calling listIngestionJobs method:", e); + responseObserver.onError( + Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling listIngestionJobs method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } + } + + @Override + public void restartIngestionJob( + RestartIngestionJobRequest request, + StreamObserver responseObserver) { + try { + RestartIngestionJobResponse response = this.jobService.restartJob(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (NoSuchElementException e) { + log.error( + "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); + responseObserver.onError( + Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + } catch (UnsupportedOperationException e) { + log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); + responseObserver.onError( + + Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling restartIngestionJob method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } + } + + @Override + public void stopIngestionJob( + StopIngestionJobRequest request, StreamObserver + responseObserver) { + try { + StopIngestionJobResponse response = this.jobService.stopJob(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (NoSuchElementException e) { + log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); + responseObserver.onError( + Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + } catch (UnsupportedOperationException e) { + log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); + responseObserver.onError( + + Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling stopIngestionJob method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } + } +} diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index 22b3fb72e09..bdb35d97b38 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -90,27 +90,29 @@ service CoreService { // Lists all projects active projects. rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse); - + + // Internal API for Job Coordinator to update featureSet's status once responsible ingestion job is running + rpc UpdateFeatureSetStatus(UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); + +} + +service JobCoordinatorService { // List Ingestion Jobs given an optional filter. - // Returns allow ingestions matching the given request filter. + // Returns allow ingestions matching the given request filter. // Returns all ingestion jobs if no filter is provided. // Returns an empty list if no ingestion jobs match the filter. rpc ListIngestionJobs(ListIngestionJobsRequest) returns (ListIngestionJobsResponse); // Restart an Ingestion Job. Restarts the ingestion job with the given job id. // NOTE: Data might be lost during the restart for some job runners. - // Does not support stopping a job in a transitional (ie pending, suspending, aborting), + // Does not support stopping a job in a transitional (ie pending, suspending, aborting), // terminal state (ie suspended or aborted) or unknown status rpc RestartIngestionJob(RestartIngestionJobRequest) returns (RestartIngestionJobResponse); - + // Stop an Ingestion Job. Stop (Aborts) the ingestion job with the given job id. // Does nothing if the target job if already in a terminal state (ie suspended or aborted). // Does not support stopping a job in a transitional (ie pending, suspending, aborting) or unknown status rpc StopIngestionJob(StopIngestionJobRequest) returns (StopIngestionJobResponse); - - // Internal API for Job Coordinator to update featureSet's status once responsible ingestion job is running - rpc UpdateFeatureSetStatus(UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); - } // Request for a single feature set diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 77976fa41d6..e9cfff4294b 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -22,6 +22,7 @@ import yaml from feast.client import Client +from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient from feast.config import Config from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.feature_set import FeatureSet, FeatureSetRef @@ -351,7 +352,7 @@ def ingest_job_list(job_id, feature_set_ref, store_name): feature_set_ref = FeatureSetRef.from_str(feature_set_ref) # pull & render ingestion jobs as a table - feast_client = Client() + feast_client = JobCoordinatorClient() table = [] for ingest_job in feast_client.list_ingest_jobs( job_id=job_id, feature_set_ref=feature_set_ref, store_name=store_name @@ -370,7 +371,7 @@ def ingest_job_describe(job_id: str): Describe the ingestion job with the given id. """ # find ingestion job for id - feast_client = Client() + feast_client = JobCoordinatorClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") @@ -399,7 +400,7 @@ def ingest_job_stop(wait: bool, timeout: int, job_id: str): Stop ingestion job for id. """ # find ingestion job for id - feast_client = Client() + feast_client = JobCoordinatorClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 86b9a2c57f6..6166df06f7c 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -58,19 +58,16 @@ ListFeatureSetsResponse, ListFeaturesRequest, ListFeaturesResponse, - ListIngestionJobsRequest, ListProjectsRequest, ListProjectsResponse, - RestartIngestionJobRequest, - StopIngestionJobRequest, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub from feast.core.FeatureSet_pb2 import FeatureSetStatus from feast.feature import Feature, FeatureRef -from feast.feature_set import Entity, FeatureSet, FeatureSetRef +from feast.feature_set import Entity, FeatureSet from feast.grpc import auth as feast_auth from feast.grpc.grpc import create_grpc_channel -from feast.job import IngestJob, RetrievalJob +from feast.job import RetrievalJob from feast.loaders.abstract_producer import get_producer from feast.loaders.file import export_source_to_staging_location from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT, get_feature_row_chunks @@ -739,78 +736,6 @@ def get_online_features( response = OnlineResponse(response) return response - def list_ingest_jobs( - self, - job_id: str = None, - feature_set_ref: FeatureSetRef = None, - store_name: str = None, - ): - """ - List the ingestion jobs currently registered in Feast, with optional filters. - Provides detailed metadata about each ingestion job. - - Args: - job_id: Select specific ingestion job with the given job_id - feature_set_ref: Filter ingestion jobs by target feature set (via reference) - store_name: Filter ingestion jobs by target feast store's name - - Returns: - List of IngestJobs matching the given filters - """ - # construct list request - feature_set_ref_proto = None - if feature_set_ref: - feature_set_ref_proto = feature_set_ref.to_proto() - list_filter = ListIngestionJobsRequest.Filter( - id=job_id, - feature_set_reference=feature_set_ref_proto, - store_name=store_name, - ) - request = ListIngestionJobsRequest(filter=list_filter) - # make list request & unpack response - response = self._core_service.ListIngestionJobs(request, metadata=self._get_grpc_metadata(),) # type: ignore - ingest_jobs = [ - IngestJob(proto, self._core_service, auth_metadata_plugin=self._auth_metadata) for proto in response.jobs # type: ignore - ] - - return ingest_jobs - - def restart_ingest_job(self, job: IngestJob): - """ - Restart ingestion job currently registered in Feast. - NOTE: Data might be lost during the restart for some job runners. - Does not support stopping a job in a transitional (ie pending, suspending, aborting), - terminal state (ie suspended or aborted) or unknown status - - Args: - job: IngestJob to restart - """ - request = RestartIngestionJobRequest(id=job.id) - try: - self._core_service.RestartIngestionJob( - request, metadata=self._get_grpc_metadata(), - ) # type: ignore - except grpc.RpcError as e: - raise grpc.RpcError(e.details()) - - def stop_ingest_job(self, job: IngestJob): - """ - Stop ingestion job currently resgistered in Feast - Does nothing if the target job if already in a terminal state (ie suspended or aborted). - Does not support stopping a job in a transitional (ie pending, suspending, aborting) - or in a unknown status - - Args: - job: IngestJob to restart - """ - request = StopIngestionJobRequest(id=job.id) - try: - self._core_service.StopIngestionJob( - request, metadata=self._get_grpc_metadata(), - ) # type: ignore - except grpc.RpcError as e: - raise grpc.RpcError(e.details()) - def ingest( self, feature_set: Union[str, FeatureSet], diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 67f4808010e..358c70b6661 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -45,6 +45,7 @@ class AuthProvider(Enum): CONFIG_ENABLE_AUTH_KEY = "enable_auth" CONFIG_ENABLE_AUTH_TOKEN_KEY = "auth_token" CONFIG_CORE_SERVER_SSL_CERT_KEY = "core_server_ssl_cert" +CONFIG_JC_SERVER_KEY = "jc_url" CONFIG_SERVING_URL_KEY = "serving_url" CONFIG_SERVING_ENABLE_SSL_KEY = "serving_enable_ssl" CONFIG_SERVING_SERVER_SSL_CERT_KEY = "serving_server_ssl_cert" @@ -75,6 +76,8 @@ class AuthProvider(Enum): CONFIG_ENABLE_AUTH_KEY: "False", # Path to certificate(s) to secure connection to Feast Core CONFIG_CORE_SERVER_SSL_CERT_KEY: "", + # Default Feast Job Coordinator URL + CONFIG_JC_SERVER_KEY: "localhost:6570", # Default Feast Serving URL CONFIG_SERVING_URL_KEY: "localhost:6565", # Enable or disable TLS/SSL to Feast Serving diff --git a/sdk/python/feast/contrib/__init__.py b/sdk/python/feast/contrib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/contrib/job_coordinator/__init__.py b/sdk/python/feast/contrib/job_coordinator/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/contrib/job_coordinator/client.py b/sdk/python/feast/contrib/job_coordinator/client.py new file mode 100644 index 00000000000..774b5c43ca7 --- /dev/null +++ b/sdk/python/feast/contrib/job_coordinator/client.py @@ -0,0 +1,140 @@ +from typing import Optional + +import grpc + +from feast.config import Config +from feast.constants import CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, CONFIG_CORE_SERVER_SSL_CERT_KEY, \ + CONFIG_ENABLE_AUTH_KEY, CONFIG_CORE_ENABLE_SSL_KEY, CONFIG_JC_SERVER_KEY +from feast.core.CoreService_pb2 import ( + ListIngestionJobsRequest, + RestartIngestionJobRequest, + StopIngestionJobRequest +) +from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub +from feast.feature_set import FeatureSetRef +from feast.grpc.grpc import create_grpc_channel +from feast.grpc import auth as feast_auth +from feast.contrib.job_coordinator.job import IngestJob + + +class Client: + """ + JobCoordinator Client: used internally to manage Ingestion Jobs + """ + + def __init__(self, options=None, **kwargs): + """ + JobCoordinatorClient should be initialized with + jc_url: Feast JobCoordinator address + + :param options: Configuration options to initialize client with + :param kwargs: options in kwargs style + """ + if options is None: + options = dict() + self._config = Config(options={**options, **kwargs}) + + self._jc_service_stub: Optional[JobCoordinatorServiceStub] = None + self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None + + # Configure Auth Metadata Plugin if auth is enabled + if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY): + self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) + + @property + def _jc_service(self): + if not self._jc_service_stub: + channel = create_grpc_channel( + url=self._config.get(CONFIG_JC_SERVER_KEY), + enable_ssl=self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY), + enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), + ssl_server_cert_path=self._config.get(CONFIG_CORE_SERVER_SSL_CERT_KEY), + auth_metadata_plugin=self._auth_metadata, + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + ) + self._jc_service_stub = JobCoordinatorServiceStub(channel) + + return self._jc_service_stub + + def list_ingest_jobs( + self, + job_id: str = None, + feature_set_ref: FeatureSetRef = None, + store_name: str = None, + ): + """ + List the ingestion jobs currently registered in Feast, with optional filters. + Provides detailed metadata about each ingestion job. + + Args: + job_id: Select specific ingestion job with the given job_id + feature_set_ref: Filter ingestion jobs by target feature set (via reference) + store_name: Filter ingestion jobs by target feast store's name + + Returns: + List of IngestJobs matching the given filters + """ + # construct list request + feature_set_ref_proto = None + if feature_set_ref: + feature_set_ref_proto = feature_set_ref.to_proto() + list_filter = ListIngestionJobsRequest.Filter( + id=job_id, + feature_set_reference=feature_set_ref_proto, + store_name=store_name, + ) + request = ListIngestionJobsRequest(filter=list_filter) + # make list request & unpack response + response = self._jc_service.ListIngestionJobs(request, metadata=self._get_grpc_metadata(),) # type: ignore + ingest_jobs = [ + IngestJob(proto, self._jc_service, auth_metadata_plugin=self._auth_metadata) for proto in response.jobs # type: ignore + ] + + return ingest_jobs + + def restart_ingest_job(self, job: IngestJob): + """ + Restart ingestion job currently registered in Feast. + NOTE: Data might be lost during the restart for some job runners. + Does not support stopping a job in a transitional (ie pending, suspending, aborting), + terminal state (ie suspended or aborted) or unknown status + + Args: + job: IngestJob to restart + """ + request = RestartIngestionJobRequest(id=job.id) + try: + self._jc_service.RestartIngestionJob( + request, metadata=self._get_grpc_metadata(), + ) # type: ignore + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + + def stop_ingest_job(self, job: IngestJob): + """ + Stop ingestion job currently resgistered in Feast + Does nothing if the target job if already in a terminal state (ie suspended or aborted). + Does not support stopping a job in a transitional (ie pending, suspending, aborting) + or in a unknown status + + Args: + job: IngestJob to restart + """ + request = StopIngestionJobRequest(id=job.id) + try: + self._jc_service.StopIngestionJob( + request, metadata=self._get_grpc_metadata(), + ) # type: ignore + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + + def _get_grpc_metadata(self): + """ + Returns a metadata tuple to attach to gRPC requests. This is primarily + used when authentication is enabled but SSL/TLS is disabled. + + Returns: Tuple of metadata to attach to each gRPC call + """ + if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY) and self._auth_metadata: + return self._auth_metadata.get_signed_meta() + return () \ No newline at end of file diff --git a/sdk/python/feast/contrib/job_coordinator/job.py b/sdk/python/feast/contrib/job_coordinator/job.py new file mode 100644 index 00000000000..4c2faadcfbf --- /dev/null +++ b/sdk/python/feast/contrib/job_coordinator/job.py @@ -0,0 +1,121 @@ +from typing import List + +import grpc +from google.protobuf.json_format import MessageToJson + +from feast import Source +from feast.core.CoreService_pb2 import ListIngestionJobsRequest +from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub +from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto, IngestionJobStatus +from feast.core.Store_pb2 import Store +from feast.feature_set import FeatureSetRef +from feast.wait import wait_retry_backoff + + +class IngestJob: + """ + Defines a job for feature ingestion in feast. + """ + + def __init__( + self, + job_proto: IngestJobProto, + core_stub: JobCoordinatorServiceStub, + auth_metadata_plugin: grpc.AuthMetadataPlugin = None, + ): + """ + Construct a native ingest job from its protobuf version. + + Args: + job_proto: Job proto object to construct from. + core_stub: stub for Feast CoreService + auth_metadata_plugin: plugin to fetch auth metadata + """ + self.proto = job_proto + self.core_svc = core_stub + self.auth_metadata = auth_metadata_plugin + + def reload(self): + """ + Update this IngestJob with the latest info from Feast + """ + # pull latest proto from feast core + response = self.core_svc.ListIngestionJobs( + ListIngestionJobsRequest( + filter=ListIngestionJobsRequest.Filter(id=self.id) + ), + metadata=self.auth_metadata.get_signed_meta() if self.auth_metadata else (), + ) + self.proto = response.jobs[0] + + @property + def id(self) -> str: + """ + Getter for IngestJob's job id. + """ + return self.proto.id + + @property + def external_id(self) -> str: + """ + Getter for IngestJob's external job id. + """ + self.reload() + return self.proto.external_id + + @property + def status(self) -> IngestionJobStatus: # type: ignore + """ + Getter for IngestJob's status + """ + self.reload() + return self.proto.status + + @property + def feature_sets(self) -> List[FeatureSetRef]: + """ + Getter for the IngestJob's feature sets + """ + # convert featureset protos to native objects + return [ + FeatureSetRef.from_proto(fs) for fs in self.proto.feature_set_references + ] + + @property + def source(self) -> Source: + """ + Getter for the IngestJob's data source. + """ + return Source.from_proto(self.proto.source) + + @property + def stores(self) -> List[Store]: + """ + Getter for the IngestJob's target feast store. + """ + return list(self.proto.stores) + + def wait(self, status: IngestionJobStatus, timeout_secs: int = 300): # type: ignore + """ + Wait for this IngestJob to transtion to the given status. + Raises TimeoutError if the wait operation times out. + + Args: + status: The IngestionJobStatus to wait for. + timeout_secs: Maximum seconds to wait before timing out. + """ + # poll & wait for job status to transition + wait_retry_backoff( + retry_fn=(lambda: (None, self.status == status)), # type: ignore + timeout_secs=timeout_secs, + timeout_msg="Wait for IngestJob's status to transition timed out", + ) + + def __str__(self): + # render the contents of ingest job as human readable string + self.reload() + return str(MessageToJson(self.proto)) + + def __repr__(self): + # render the ingest job as human readable string + return f"IngestJob<{self.id}>" \ No newline at end of file diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index f7ea43dfaa1..2a45207cbff 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -4,16 +4,9 @@ import fastavro import grpc import pandas as pd -from google.protobuf.json_format import MessageToJson from feast.constants import CONFIG_TIMEOUT_KEY from feast.constants import FEAST_DEFAULT_OPTIONS as defaults -from feast.core.CoreService_pb2 import ListIngestionJobsRequest -from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto -from feast.core.IngestionJob_pb2 import IngestionJobStatus -from feast.core.Store_pb2 import Store -from feast.feature_set import FeatureSetRef from feast.serving.ServingService_pb2 import ( DATA_FORMAT_AVRO, JOB_STATUS_DONE, @@ -21,7 +14,6 @@ ) from feast.serving.ServingService_pb2 import Job as JobProto from feast.serving.ServingService_pb2_grpc import ServingServiceStub -from feast.source import Source from feast.staging.storage_client import get_staging_client from feast.wait import wait_retry_backoff from tensorflow_metadata.proto.v0 import statistics_pb2 @@ -220,112 +212,3 @@ def statistics( if self.job_proto.error: raise Exception(self.job_proto.error) return self.job_proto.dataset_feature_statistics_list - - -class IngestJob: - """ - Defines a job for feature ingestion in feast. - """ - - def __init__( - self, - job_proto: IngestJobProto, - core_stub: CoreServiceStub, - auth_metadata_plugin: grpc.AuthMetadataPlugin = None, - ): - """ - Construct a native ingest job from its protobuf version. - - Args: - job_proto: Job proto object to construct from. - core_stub: stub for Feast CoreService - auth_metadata_plugin: plugin to fetch auth metadata - """ - self.proto = job_proto - self.core_svc = core_stub - self.auth_metadata = auth_metadata_plugin - - def reload(self): - """ - Update this IngestJob with the latest info from Feast - """ - # pull latest proto from feast core - response = self.core_svc.ListIngestionJobs( - ListIngestionJobsRequest( - filter=ListIngestionJobsRequest.Filter(id=self.id) - ), - metadata=self.auth_metadata.get_signed_meta() if self.auth_metadata else (), - ) - self.proto = response.jobs[0] - - @property - def id(self) -> str: - """ - Getter for IngestJob's job id. - """ - return self.proto.id - - @property - def external_id(self) -> str: - """ - Getter for IngestJob's external job id. - """ - self.reload() - return self.proto.external_id - - @property - def status(self) -> IngestionJobStatus: # type: ignore - """ - Getter for IngestJob's status - """ - self.reload() - return self.proto.status - - @property - def feature_sets(self) -> List[FeatureSetRef]: - """ - Getter for the IngestJob's feature sets - """ - # convert featureset protos to native objects - return [ - FeatureSetRef.from_proto(fs) for fs in self.proto.feature_set_references - ] - - @property - def source(self) -> Source: - """ - Getter for the IngestJob's data source. - """ - return Source.from_proto(self.proto.source) - - @property - def stores(self) -> List[Store]: - """ - Getter for the IngestJob's target feast store. - """ - return list(self.proto.stores) - - def wait(self, status: IngestionJobStatus, timeout_secs: int = 300): # type: ignore - """ - Wait for this IngestJob to transtion to the given status. - Raises TimeoutError if the wait operation times out. - - Args: - status: The IngestionJobStatus to wait for. - timeout_secs: Maximum seconds to wait before timing out. - """ - # poll & wait for job status to transition - wait_retry_backoff( - retry_fn=(lambda: (None, self.status == status)), # type: ignore - timeout_secs=timeout_secs, - timeout_msg="Wait for IngestJob's status to transition timed out", - ) - - def __str__(self): - # render the contents of ingest job as human readable string - self.reload() - return str(MessageToJson(self.proto)) - - def __repr__(self): - # render the ingest job as human readable string - return f"IngestJob<{self.id}>" diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 810b7ccc595..bca95a664b2 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -28,6 +28,7 @@ from pytz import timezone from feast.client import Client +from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient from feast.core import CoreService_pb2_grpc as Core from feast.core.CoreService_pb2 import ( GetFeastCoreVersionResponse, @@ -49,7 +50,7 @@ from feast.entity import Entity from feast.feature import Feature from feast.feature_set import FeatureSet, FeatureSetRef -from feast.job import IngestJob +from feast.contrib.job_coordinator.job import IngestJob from feast.serving import ServingService_pb2_grpc as Serving from feast.serving.ServingService_pb2 import DataFormat, FeastServingType from feast.serving.ServingService_pb2 import FeatureReference as FeatureRefProto @@ -74,6 +75,7 @@ CORE_URL = "core.feast.example.com" SERVING_URL = "serving.example.com" +JC_URL = "jc.feast.example.com" _PRIVATE_KEY_RESOURCE_PATH = "data/localhost.key" _CERTIFICATE_CHAIN_RESOURCE_PATH = "data/localhost.pem" _ROOT_CERTIFICATE_RESOURCE_PATH = "data/localhost.crt" @@ -105,6 +107,11 @@ def mock_client(self): client._serving_url = SERVING_URL return client + @pytest.fixture + def mock_jc_client(self): + client = JobCoordinatorClient(jc_url=JC_URL) + return client + @pytest.fixture def mock_client_with_auth(self): client = Client( @@ -541,21 +548,17 @@ def test_list_features(self, mocked_client, mocker): and set(feature_dtype_list) == set([ValueType.FLOAT, ValueType.STRING]) ) - @pytest.mark.parametrize( - "mocked_client", - [lazy_fixture("mock_client"), lazy_fixture("secure_mock_client")], - ) - def test_list_ingest_jobs(self, mocked_client, mocker): + def test_list_ingest_jobs(self, mock_jc_client, mocker): mocker.patch.object( - mocked_client, - "_core_service_stub", - return_value=Core.CoreServiceStub(grpc.insecure_channel("")), + mock_jc_client, + "_jc_service_stub", + return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), ) feature_set_ref = FeatureSetRef(project="test", name="driver",) mocker.patch.object( - mocked_client._core_service_stub, + mock_jc_client._jc_service_stub, "ListIngestionJobs", return_value=ListIngestionJobsResponse( jobs=[ @@ -577,7 +580,7 @@ def test_list_ingest_jobs(self, mocked_client, mocker): ) # list ingestion jobs by target feature set reference - ingest_jobs = mocked_client.list_ingest_jobs(feature_set_ref=feature_set_ref) + ingest_jobs = mock_jc_client.list_ingest_jobs(feature_set_ref=feature_set_ref) assert len(ingest_jobs) >= 1 ingest_job = ingest_jobs[0] @@ -589,15 +592,11 @@ def test_list_ingest_jobs(self, mocked_client, mocker): and ingest_job.source.source_type == "Kafka" ) - @pytest.mark.parametrize( - "mocked_client", - [lazy_fixture("mock_client"), lazy_fixture("secure_mock_client")], - ) - def test_restart_ingest_job(self, mocked_client, mocker): + def test_restart_ingest_job(self, mock_jc_client, mocker): mocker.patch.object( - mocked_client, - "_core_service_stub", - return_value=Core.CoreServiceStub(grpc.insecure_channel("")), + mock_jc_client, + "_jc_service_stub", + return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), ) ingest_job = IngestJob( @@ -606,21 +605,17 @@ def test_restart_ingest_job(self, mocked_client, mocker): external_id="job#2222", status=IngestionJobStatus.ERROR, ), - core_stub=mocked_client._core_service_stub, + core_stub=mock_jc_client._jc_service_stub, ) - mocked_client.restart_ingest_job(ingest_job) - assert mocked_client._core_service_stub.RestartIngestionJob.called + mock_jc_client.restart_ingest_job(ingest_job) + assert mock_jc_client._jc_service_stub.RestartIngestionJob.called - @pytest.mark.parametrize( - "mocked_client", - [lazy_fixture("mock_client"), lazy_fixture("secure_mock_client")], - ) - def test_stop_ingest_job(self, mocked_client, mocker): + def test_stop_ingest_job(self, mock_jc_client, mocker): mocker.patch.object( - mocked_client, - "_core_service_stub", - return_value=Core.CoreServiceStub(grpc.insecure_channel("")), + mock_jc_client, + "_jc_service_stub", + return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), ) ingest_job = IngestJob( @@ -629,11 +624,11 @@ def test_stop_ingest_job(self, mocked_client, mocker): external_id="job#2222", status=IngestionJobStatus.RUNNING, ), - core_stub=mocked_client._core_service_stub, + core_stub=mock_jc_client._jc_service_stub, ) - mocked_client.stop_ingest_job(ingest_job) - assert mocked_client._core_service_stub.StopIngestionJob.called + mock_jc_client.stop_ingest_job(ingest_job) + assert mock_jc_client._jc_service_stub.StopIngestionJob.called @pytest.mark.parametrize( "mocked_client", diff --git a/storage/connectors/bigquery/pom.xml b/storage/connectors/bigquery/pom.xml index 1b97d57b2cb..bb304452ac6 100644 --- a/storage/connectors/bigquery/pom.xml +++ b/storage/connectors/bigquery/pom.xml @@ -58,13 +58,6 @@ 1.6.6 - - com.google.auto.value - auto-value - 1.6.6 - provided - - junit junit diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java index 8a9a5995b62..3a07092fbcc 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java @@ -24,8 +24,6 @@ import feast.proto.types.FeatureRowProto.FeatureRow; import feast.storage.api.writer.FailedElement; import feast.storage.api.writer.WriteResult; -import feast.storage.connectors.bigquery.compression.CompactFeatureRows; -import feast.storage.connectors.bigquery.compression.FeatureRowsBatch; import java.util.List; import java.util.Map; import org.apache.beam.sdk.coders.KvCoder; @@ -52,7 +50,8 @@ public class BigQueryWrite extends PTransform, WriteResu private static final Duration BIGQUERY_DEFAULT_WRITE_TRIGGERING_FREQUENCY = Duration.standardMinutes(1); - private static final Duration BIGQUERY_JOB_MAX_EXPECTING_RESULT_TIME = Duration.standardHours(1); + private static final Duration BIGQUERY_JOB_MAX_EXPECTING_RESULT_TIME = + Duration.standardMinutes(10); private static final int BIGQUERY_MAX_JOB_RETRIES = 20; private static final int DEFAULT_COMPACTION_BATCH_SIZE = 10000; private static final int MAX_SUCCESSFUL_OUTPUTS_PER_DESTINATION = 1000; @@ -180,10 +179,10 @@ public void processElement(ProcessContext context) {} private PCollection mergeInputWithResult( PCollection inputInFixedWindow, PCollection> successful) { - final TupleTag inputTag = new TupleTag<>(); + final TupleTag> inputTag = new TupleTag<>(); final TupleTag successTag = new TupleTag<>(); - PCollection> insertedRows = + PCollection>> insertedRows = inputInFixedWindow .apply( "MakeElementKey", @@ -200,7 +199,7 @@ public void process(ProcessContext c) { element)); } })) - .apply(new CompactFeatureRows(compactionBatchSize)); + .apply(Sample.fixedSizePerKey(compactionBatchSize)); PCollection> inputWithResult = KeyedPCollectionTuple.of(inputTag, insertedRows) @@ -230,12 +229,7 @@ public void process(ProcessContext c) { return; } - result - .getAll(inputTag) - .forEach( - rows -> - rows.getFeatureRowsSample(maxSuccessfulOutputs) - .forEachRemaining(c::output)); + result.getAll(inputTag).forEach(rows -> rows.forEach(c::output)); } })); } From e161eee8633294c284bc1a07b62dd32742f8ab94 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 12:10:56 +0800 Subject: [PATCH 04/26] infra fix simple client --- .github/workflows/complete.yml | 2 +- .github/workflows/master_only.yml | 2 +- Makefile | 9 +- .../src/main/java/feast/common/it/BaseIT.java | 6 +- .../java/feast/common/it/ExternalApp.java | 92 ++++++++++- ...leAPIClient.java => SimpleCoreClient.java} | 17 +- .../java/feast/common/it/SimpleJCClient.java | 43 +++++ core/pom.xml | 6 - .../java/feast/core/grpc/CoreServiceImpl.java | 71 -------- .../auth/CoreServiceAuthenticationIT.java | 18 +- .../core/auth/CoreServiceAuthorizationIT.java | 23 +-- .../feast/core/service/SpecServiceIT.java | 6 +- .../charts/feast/charts/feast-core/Chart.yaml | 2 +- .../charts/feast/charts/feast-core/README.md | 2 +- .../feast-core/templates/configmap.yaml | 6 - infra/charts/feast/charts/feast-jc/Chart.yaml | 4 + infra/charts/feast/charts/feast-jc/README.md | 71 ++++++++ .../charts/feast-jc/templates/_helpers.tpl | 45 +++++ .../charts/feast-jc/templates/_ingress.yaml | 68 ++++++++ .../charts/feast-jc/templates/configmap.yaml | 35 ++++ .../charts/feast-jc/templates/deployment.yaml | 152 +++++++++++++++++ .../charts/feast-jc/templates/ingress.yaml | 7 + .../charts/feast-jc/templates/secret.yaml | 15 ++ .../charts/feast-jc/templates/service.yaml | 41 +++++ .../charts/feast/charts/feast-jc/values.yaml | 156 ++++++++++++++++++ .../charts/feast/values-dataflow-runner.yaml | 9 + infra/charts/feast/values.yaml | 4 + infra/docker/core/Dockerfile | 14 +- infra/docker/jc/Dockerfile | 72 ++++++++ infra/docker/jc/Dockerfile.debug | 0 infra/docker/jc/Dockerfile.dev | 8 + infra/scripts/setup-common-functions.sh | 15 ++ infra/scripts/test-end-to-end-batch.sh | 7 +- .../scripts/test-end-to-end-redis-cluster.sh | 7 +- infra/scripts/test-end-to-end.sh | 17 +- .../values-end-to-end-batch-dataflow.yaml | 14 ++ job-coordinator/pom.xml | 26 +-- .../feast/jc/dao/InMemoryJobRepository.java | 5 +- .../java/feast/jc/grpc/CoreServiceImpl.java | 144 +++++++--------- .../jc/model/FeatureSetDeliveryStatus.java | 32 +++- .../src/main/java/feast/jc/model/Job.java | 3 +- .../jc/service/JobCoordinatorService.java | 80 +++++---- .../src/main/resources/application.yml | 4 +- .../feast/jc/service/JobCoordinatorIT.java | 103 +++++++----- .../java/feast/jc/service/JobServiceIT.java | 8 +- .../test/resources/application-it-core.yml | 2 - pom.xml | 2 +- sdk/python/feast/cli.py | 4 +- .../feast/contrib/job_coordinator/client.py | 19 ++- .../feast/contrib/job_coordinator/job.py | 5 +- sdk/python/tests/test_client.py | 2 +- tests/e2e/bq/bq-batch-retrieval.py | 11 +- tests/e2e/redis/basic-ingest-redis-serving.py | 32 ++-- 53 files changed, 1183 insertions(+), 365 deletions(-) rename common-test/src/main/java/feast/common/it/{SimpleAPIClient.java => SimpleCoreClient.java} (89%) create mode 100644 common-test/src/main/java/feast/common/it/SimpleJCClient.java create mode 100644 infra/charts/feast/charts/feast-jc/Chart.yaml create mode 100644 infra/charts/feast/charts/feast-jc/README.md create mode 100644 infra/charts/feast/charts/feast-jc/templates/_helpers.tpl create mode 100644 infra/charts/feast/charts/feast-jc/templates/_ingress.yaml create mode 100644 infra/charts/feast/charts/feast-jc/templates/configmap.yaml create mode 100644 infra/charts/feast/charts/feast-jc/templates/deployment.yaml create mode 100644 infra/charts/feast/charts/feast-jc/templates/ingress.yaml create mode 100644 infra/charts/feast/charts/feast-jc/templates/secret.yaml create mode 100644 infra/charts/feast/charts/feast-jc/templates/service.yaml create mode 100644 infra/charts/feast/charts/feast-jc/values.yaml create mode 100644 infra/docker/jc/Dockerfile create mode 100644 infra/docker/jc/Dockerfile.debug create mode 100644 infra/docker/jc/Dockerfile.dev diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 145bab5ed53..9300227bf51 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -7,7 +7,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jupyter] + component: [core, serving, jc, jupyter] env: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} REGISTRY: gcr.io/kf-feast diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 3ee54cf2c71..424b670f601 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jupyter, ci] + component: [core, serving, jc, jupyter, ci] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar steps: diff --git a/Makefile b/Makefile index 1ed4f1eb721..8af2343c59f 100644 --- a/Makefile +++ b/Makefile @@ -117,12 +117,16 @@ build-push-docker: @$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION) @$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION) @$(MAKE) push-ci-docker registry=$(REGISTRY) version=$(VERSION) + @$(MAKE) push-jc-docker registry=$(REGISTRY) version=$(VERSION) -build-docker: build-core-docker build-serving-docker build-ci-docker +build-docker: build-core-docker build-serving-docker build-ci-docker build-jc-docker push-core-docker: docker push $(REGISTRY)/feast-core:$(VERSION) +push-jc-docker: + docker push $(REGISTRY)/feast-jc:$(VERSION) + push-serving-docker: docker push $(REGISTRY)/feast-serving:$(VERSION) @@ -135,6 +139,9 @@ push-jupyter-docker: build-core-docker: docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . +build-jc-docker: + docker build -t $(REGISTRY)/feast-jc:$(VERSION) -f infra/docker/jc/Dockerfile . + build-serving-docker: docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java index 0a98043860a..c6001f209cc 100644 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -35,7 +35,6 @@ import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.KafkaContainer; @@ -51,7 +50,6 @@ @ActiveProfiles("it") @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -@ContextConfiguration public class BaseIT { @Container public static PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer<>(); @@ -135,6 +133,10 @@ public static void cleanTables() throws SQLException { tableNames.add(rs.getString(1)); } + if (tableNames.isEmpty()) { + return; + } + // retries are needed since truncate require exclusive lock // and that often leads to Deadlock // since SpringApp is still running in another thread diff --git a/common-test/src/main/java/feast/common/it/ExternalApp.java b/common-test/src/main/java/feast/common/it/ExternalApp.java index cd8a3c44edf..3fa98b0d502 100644 --- a/common-test/src/main/java/feast/common/it/ExternalApp.java +++ b/common-test/src/main/java/feast/common/it/ExternalApp.java @@ -1,2 +1,92 @@ -package feast.common.it;public class ExternalApp { +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.common.it; + +import com.google.auto.value.AutoValue; +import java.util.HashMap; +import java.util.Map; +import org.springframework.boot.Banner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.util.SocketUtils; +import org.testcontainers.containers.PostgreSQLContainer; + +@AutoValue +public abstract class ExternalApp { + private ConfigurableApplicationContext appContext; + + abstract PostgreSQLContainer getPostgreSQL(); + + abstract String getName(); + + abstract int getGRPCPort(); + + abstract int getWebPort(); + + abstract Map getProperties(); + + abstract Class getSpringApplication(); + + public static Builder builder() { + return new AutoValue_ExternalApp.Builder() + .setProperties(new HashMap<>()) + .setWebPort(SocketUtils.findAvailableTcpPort()); + } + + @AutoValue.Builder + public interface Builder { + Builder setSpringApplication(Class app); + + Builder setName(String name); + + Builder setPostgreSQL(PostgreSQLContainer psql); + + Builder setGRPCPort(int port); + + Builder setWebPort(int port); + + Builder setProperties(Map properties); + + ExternalApp build(); + } + + public void start() { + HashMap properties = new HashMap<>(getProperties()); + properties.put("spring.datasource.url", getPostgreSQL().getJdbcUrl()); + properties.put("spring.datasource.username", getPostgreSQL().getUsername()); + properties.put("spring.datasource.password", getPostgreSQL().getPassword()); + properties.put("grpc.server.port", getGRPCPort()); + properties.put("server.port", getWebPort()); + + StandardEnvironment env = new StandardEnvironment(); + env.setDefaultProfiles(getName()); + env.getPropertySources().addFirst(new MapPropertySource("primary", properties)); + + appContext = + new SpringApplicationBuilder(getSpringApplication()) + .environment(env) + .bannerMode(Banner.Mode.OFF) + .run(); + } + + public void stop() { + SpringApplication.exit(appContext); + } } diff --git a/common-test/src/main/java/feast/common/it/SimpleAPIClient.java b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java similarity index 89% rename from common-test/src/main/java/feast/common/it/SimpleAPIClient.java rename to common-test/src/main/java/feast/common/it/SimpleCoreClient.java index c9855facdf1..4b38886b1bd 100644 --- a/common-test/src/main/java/feast/common/it/SimpleAPIClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java @@ -22,10 +22,10 @@ import java.util.List; import java.util.Map; -public class SimpleAPIClient { +public class SimpleCoreClient { private CoreServiceGrpc.CoreServiceBlockingStub stub; - public SimpleAPIClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { + public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { this.stub = stub; } @@ -127,19 +127,6 @@ public void archiveProject(String name) { stub.archiveProject(CoreServiceProto.ArchiveProjectRequest.newBuilder().setName(name).build()); } - public void restartIngestionJob(String jobId) { - stub.restartIngestionJob( - CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(jobId).build()); - } - - public List listIngestionJobs() { - return stub.listIngestionJobs( - CoreServiceProto.ListIngestionJobsRequest.newBuilder() - .setFilter(CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build()) - .build()) - .getJobsList(); - } - public String getFeastCoreVersion() { return stub.getFeastCoreVersion( CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance()) diff --git a/common-test/src/main/java/feast/common/it/SimpleJCClient.java b/common-test/src/main/java/feast/common/it/SimpleJCClient.java new file mode 100644 index 00000000000..99bfb349c56 --- /dev/null +++ b/common-test/src/main/java/feast/common/it/SimpleJCClient.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.common.it; + +import feast.proto.core.CoreServiceProto; +import feast.proto.core.IngestionJobProto; +import feast.proto.core.JobCoordinatorServiceGrpc; +import java.util.List; + +public class SimpleJCClient { + private final JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub; + + public SimpleJCClient(JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub) { + this.stub = stub; + } + + public void restartIngestionJob(String jobId) { + stub.restartIngestionJob( + CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(jobId).build()); + } + + public List listIngestionJobs() { + return stub.listIngestionJobs( + CoreServiceProto.ListIngestionJobsRequest.newBuilder() + .setFilter(CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build()) + .build()) + .getJobsList(); + } +} diff --git a/core/pom.xml b/core/pom.xml index aaf24e2cc49..483c23d550c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -305,12 +305,6 @@ 6.1.2.Final - - dev.feast - feast-common-test - ${project.version} - test - sh.ory.keto keto-client diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index ab540c08949..ea114005bca 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -47,7 +47,6 @@ public class CoreServiceImpl extends CoreServiceImplBase { private final FeastProperties feastProperties; private SpecService specService; - // private JobService jobService; private StatsService statsService; private ProjectService projectService; private final AuthorizationService authorizationService; @@ -57,12 +56,10 @@ public CoreServiceImpl( SpecService specService, ProjectService projectService, StatsService statsService, - // JobService jobService, FeastProperties feastProperties, AuthorizationService authorizationService) { this.specService = specService; this.projectService = projectService; - // this.jobService = jobService; this.feastProperties = feastProperties; this.statsService = statsService; this.authorizationService = authorizationService; @@ -307,72 +304,4 @@ public void listProjects( Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } } - - // @Override - // public void listIngestionJobs( - // ListIngestionJobsRequest request, - // StreamObserver responseObserver) { - // try { - // ListIngestionJobsResponse response = this.jobService.listJobs(request); - // responseObserver.onNext(response); - // responseObserver.onCompleted(); - // } catch (InvalidArgumentException e) { - // log.error("Recieved an invalid request on calling listIngestionJobs method:", e); - // responseObserver.onError( - // Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); - // } catch (Exception e) { - // log.error("Unexpected exception on calling listIngestionJobs method:", e); - // responseObserver.onError( - // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - // } - // } - // - // @Override - // public void restartIngestionJob( - // RestartIngestionJobRequest request, - // StreamObserver responseObserver) { - // try { - // RestartIngestionJobResponse response = this.jobService.restartJob(request); - // responseObserver.onNext(response); - // responseObserver.onCompleted(); - // } catch (NoSuchElementException e) { - // log.error( - // "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); - // responseObserver.onError( - // Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - // } catch (UnsupportedOperationException e) { - // log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); - // responseObserver.onError( - // - // Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - // } catch (Exception e) { - // log.error("Unexpected exception on calling restartIngestionJob method:", e); - // responseObserver.onError( - // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - // } - // } - - // @Override - // public void stopIngestionJob( - // StopIngestionJobRequest request, StreamObserver - // responseObserver) { - // try { - // StopIngestionJobResponse response = this.jobService.stopJob(request); - // responseObserver.onNext(response); - // responseObserver.onCompleted(); - // } catch (NoSuchElementException e) { - // log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); - // responseObserver.onError( - // Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - // } catch (UnsupportedOperationException e) { - // log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); - // responseObserver.onError( - // - // Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - // } catch (Exception e) { - // log.error("Unexpected exception on calling stopIngestionJob method:", e); - // responseObserver.onError( - // Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - // } - // } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index 9f62ea98d90..ec8abbbb3db 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -24,10 +24,9 @@ import com.nimbusds.jose.jwk.JWKSet; import feast.common.it.BaseIT; import feast.common.it.DataGenerator; -import feast.common.it.SimpleAPIClient; +import feast.common.it.SimpleCoreClient; import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; -import feast.core.model.*; import feast.proto.core.*; import io.grpc.CallCredentials; import io.grpc.Channel; @@ -68,7 +67,7 @@ public class CoreServiceAuthenticationIT extends BaseIT { @Rule public WireMockClassRule instanceRule = wireMockRule; - static SimpleAPIClient insecureApiClient; + static SimpleCoreClient insecureApiClient; @DynamicPropertySource static void initialize(DynamicPropertyRegistry registry) { @@ -104,7 +103,7 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleAPIClient(insecureCoreService); + insecureApiClient = new SimpleCoreClient(insecureCoreService); } @AfterAll @@ -114,7 +113,8 @@ static void tearDown() { @Test public void shouldGetVersionFromFeastCoreAlways() { - SimpleAPIClient secureApiClient = getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); + SimpleCoreClient secureApiClient = + getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); @@ -145,7 +145,7 @@ public void shouldAllowUnauthenticatedFeatureSetListing() { @Test public void shouldAllowAuthenticatedFeatureSetListing() { - SimpleAPIClient secureApiClient = + SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); @@ -162,7 +162,7 @@ public void shouldAllowAuthenticatedFeatureSetListing() { @Test void canApplyFeatureSetIfAuthenticated() { - SimpleAPIClient secureApiClient = + SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "project_1", "test_1"); @@ -187,7 +187,7 @@ public Map testMap() { } // Create secure Feast Core gRPC client for a specific user - private static SimpleAPIClient getSecureApiClient(String subjectEmail) { + private static SimpleCoreClient getSecureApiClient(String subjectEmail) { CallCredentials callCredentials = null; try { callCredentials = jwtHelper.getCallCredentials(subjectEmail); @@ -201,6 +201,6 @@ private static SimpleAPIClient getSecureApiClient(String subjectEmail) { CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - return new SimpleAPIClient(secureCoreService); + return new SimpleCoreClient(secureCoreService); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java index ee27b72fa46..d90296d99cf 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java @@ -26,7 +26,7 @@ import com.nimbusds.jose.jwk.JWKSet; import feast.common.it.BaseIT; import feast.common.it.DataGenerator; -import feast.common.it.SimpleAPIClient; +import feast.common.it.SimpleCoreClient; import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; import feast.proto.core.CoreServiceGrpc; @@ -82,7 +82,7 @@ public class CoreServiceAuthorizationIT extends BaseIT { static String subjectIsAdmin = "bossman@example.com"; static String subjectClaim = "sub"; - static SimpleAPIClient insecureApiClient; + static SimpleCoreClient insecureApiClient; @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); @@ -146,12 +146,12 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleAPIClient(insecureCoreService); + insecureApiClient = new SimpleCoreClient(insecureCoreService); } @BeforeEach public void setUp() { - SimpleAPIClient secureApiClient = getSecureApiClient(subjectIsAdmin); + SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); } @@ -164,7 +164,8 @@ static void tearDown() { @Test public void shouldGetVersionFromFeastCoreAlways() { - SimpleAPIClient secureApiClient = getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); + SimpleCoreClient secureApiClient = + getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); @@ -189,7 +190,7 @@ public void shouldNotAllowUnauthenticatedFeatureSetListing() { @Test public void shouldAllowAuthenticatedFeatureSetListing() { - SimpleAPIClient secureApiClient = + SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); List listFeatureSetsResponse = @@ -206,7 +207,7 @@ public void shouldAllowAuthenticatedFeatureSetListing() { @Test void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferException { String userName = "random_user@example.com"; - SimpleAPIClient secureApiClient = getSecureApiClient(userName); + SimpleCoreClient secureApiClient = getSecureApiClient(userName); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_5"); @@ -224,7 +225,7 @@ void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferExcepti @Test void canApplyFeatureSetIfProjectMember() { - SimpleAPIClient secureApiClient = getSecureApiClient(subjectInProject); + SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_6"); @@ -241,7 +242,7 @@ void canApplyFeatureSetIfProjectMember() { @Test void canApplyFeatureSetIfAdmin() { - SimpleAPIClient secureApiClient = getSecureApiClient(subjectIsAdmin); + SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "any_project", "test_2"); @@ -319,7 +320,7 @@ private static OryAccessControlPolicy getMyProjectMemberPolicy() { } // Create secure Feast Core gRPC client for a specific user - private static SimpleAPIClient getSecureApiClient(String subjectEmail) { + private static SimpleCoreClient getSecureApiClient(String subjectEmail) { CallCredentials callCredentials = null; try { callCredentials = jwtHelper.getCallCredentials(subjectEmail); @@ -333,6 +334,6 @@ private static SimpleAPIClient getSecureApiClient(String subjectEmail) { CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - return new SimpleAPIClient(secureCoreService); + return new SimpleCoreClient(secureCoreService); } } diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 66d1fb425aa..75933d3ee21 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -32,7 +32,7 @@ import com.google.protobuf.Duration; import feast.common.it.BaseIT; import feast.common.it.DataGenerator; -import feast.common.it.SimpleAPIClient; +import feast.common.it.SimpleCoreClient; import feast.proto.core.*; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; @@ -54,14 +54,14 @@ public class SpecServiceIT extends BaseIT { static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleAPIClient apiClient; + static SimpleCoreClient apiClient; @BeforeAll public static void globalSetUp(@Value("${grpc.server.port}") int port) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleAPIClient(stub); + apiClient = new SimpleCoreClient(stub); } @BeforeEach diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml index ceec1f82bfe..56ab15e7fbc 100644 --- a/infra/charts/feast/charts/feast-core/Chart.yaml +++ b/infra/charts/feast/charts/feast-core/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 -description: Feast Core registers feature specifications and manage ingestion jobs. +description: Feast Core registers feature specifications. name: feast-core version: 0.7-SNAPSHOT diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md index 01fd3a84331..36d963dd242 100644 --- a/infra/charts/feast/charts/feast-core/README.md +++ b/infra/charts/feast/charts/feast-core/README.md @@ -1,6 +1,6 @@ feast-core ========== -Feast Core registers feature specifications and manage ingestion jobs. +Feast Core registers feature specifications. Current chart version is `0.7-SNAPSHOT` diff --git a/infra/charts/feast/charts/feast-core/templates/configmap.yaml b/infra/charts/feast/charts/feast-core/templates/configmap.yaml index b48e15cc985..71c303d4300 100644 --- a/infra/charts/feast/charts/feast-core/templates/configmap.yaml +++ b/infra/charts/feast/charts/feast-core/templates/configmap.yaml @@ -21,12 +21,6 @@ data: options: bootstrapServers: {{ .Release.Name }}-kafka:9092 topic: feast - jobs: - metrics: - enabled: true - type: statsd - host: {{ .Release.Name }}-prometheus-statsd-exporter-udp - port: 9125 server: port: {{ .Values.service.http.targetPort }} diff --git a/infra/charts/feast/charts/feast-jc/Chart.yaml b/infra/charts/feast/charts/feast-jc/Chart.yaml new file mode 100644 index 00000000000..b10c7a811b9 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Feast Job Coordinator manage ingestion jobs. +name: feast-jc +version: 0.7-SNAPSHOT diff --git a/infra/charts/feast/charts/feast-jc/README.md b/infra/charts/feast/charts/feast-jc/README.md new file mode 100644 index 00000000000..8860b83bcb9 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/README.md @@ -0,0 +1,71 @@ +feast-jc +========== +Feast Job Coordinator manage ingestion jobs. + +Current chart version is `0.7-SNAPSHOT` + + + + + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. | +| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | +| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | +| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | +| envOverrides | object | `{}` | Extra environment variables to set | +| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow | +| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key | +| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | +| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"gcr.io/kf-feast/feast-jc"` | Docker image repository | +| image.tag | string | `"0.6.2"` | Image tag | +| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | +| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges | +| ingress.http.annotations | object | `{}` | Extra annotations for the ingress | +| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service | +| ingress.http.auth.enabled | bool | `false` | Flag to enable auth | +| ingress.http.class | string | `"nginx"` | Which ingress controller to use | +| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service | +| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests | +| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS | +| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name | +| ingress.http.whitelist | string | `""` | Allowed client IP source ranges | +| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` | +| livenessProbe.enabled | bool | `false` | Flag to enabled the probe | +| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated | +| livenessProbe.periodSeconds | int | `10` | How often to perform the probe | +| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| livenessProbe.timeoutSeconds | int | `5` | When the probe times out | +| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | +| logType | string | `"Console"` | Log format, either `JSON` or `Console` | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| podLabels | object | `{}` | Labels to be added to Feast Core pods | +| postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres | +| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | +| readinessProbe.enabled | bool | `true` | Flag to enabled the probe | +| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | +| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated | +| readinessProbe.periodSeconds | int | `10` | How often to perform the probe | +| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | +| readinessProbe.timeoutSeconds | int | `10` | When the probe times out | +| replicaCount | int | `1` | Number of pods that will be created | +| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) | +| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.grpc.port | int | `6565` | Service port for GRPC requests | +| service.grpc.targetPort | int | `6565` | Container port serving GRPC requests | +| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to | +| service.http.port | int | `80` | Service port for HTTP requests | +| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics | +| service.type | string | `"ClusterIP"` | Kubernetes service type | diff --git a/infra/charts/feast/charts/feast-jc/templates/_helpers.tpl b/infra/charts/feast/charts/feast-jc/templates/_helpers.tpl new file mode 100644 index 00000000000..8e18ba61d62 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "feast-jc.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "feast-jc.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "feast-jc.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "feast-jc.labels" -}} +app.kubernetes.io/name: {{ include "feast-jc.name" . }} +helm.sh/chart: {{ include "feast-jc.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/infra/charts/feast/charts/feast-jc/templates/_ingress.yaml b/infra/charts/feast/charts/feast-jc/templates/_ingress.yaml new file mode 100644 index 00000000000..5bed6df0470 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/_ingress.yaml @@ -0,0 +1,68 @@ +{{- /* +This takes an array of three values: +- the top context +- the feast component +- the service protocol +- the ingress context +*/ -}} +{{- define "feast.ingress" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +apiVersion: extensions/v1beta1 +kind: Ingress +{{ include "feast.ingress.metadata" . }} +spec: + rules: + {{- range $host := $ingressValues.hosts }} + - host: {{ $host }} + http: + paths: + - path: / + backend: + serviceName: {{ include (printf "feast-%s.fullname" $component) $top }} + servicePort: {{ index $top.Values "service" $protocol "port" }} + {{- end }} +{{- if $ingressValues.https.enabled }} + tls: + {{- range $host := $ingressValues.hosts }} + - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }} + hosts: + - {{ $host }} + {{- end }} +{{- end -}} +{{- end -}} + +{{- define "feast.ingress.metadata" -}} +{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }} +{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}} +{{- toYaml (merge $overrides $commonMetadata) -}} +{{- end -}} + +{{- define "feast.ingress.metadata-overrides" -}} +{{- $top := (index . 0) -}} +{{- $component := (index . 1) -}} +{{- $protocol := (index . 2) -}} +{{- $ingressValues := (index . 3) -}} +{{- $commonFullname := include "common.fullname" $top }} +metadata: + name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }} + annotations: + kubernetes.io/ingress.class: {{ $ingressValues.class | quote }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }} + nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }} + nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user" + nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri" + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}} + {{- end }} + {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }} + # TODO: Allow choice of GRPC/GRPCS + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + {{- end }} + {{- if $ingressValues.annotations -}} + {{ include "common.annote" $ingressValues.annotations | indent 4 }} + {{- end }} +{{- end -}} diff --git a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml new file mode 100644 index 00000000000..196dc5aa989 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-jc.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-jc.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + application-generated.yaml: | +{{- if index .Values "application-generated.yaml" "enabled" }} + feast: + stream: + type: kafka + options: + bootstrapServers: {{ .Release.Name }}-kafka:9092 + topic: feast + jobs: + metrics: + enabled: true + type: statsd + host: {{ .Release.Name }}-prometheus-statsd-exporter-udp + port: 9125 + + server: + port: {{ .Values.service.http.targetPort }} +{{- end }} + + application-override.yaml: | +{{- if index .Values "application-override.yaml" "enabled" }} +{{- toYaml (index .Values "application-override.yaml") | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/infra/charts/feast/charts/feast-jc/templates/deployment.yaml b/infra/charts/feast/charts/feast-jc/templates/deployment.yaml new file mode 100644 index 00000000000..151d4ad6c3c --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/deployment.yaml @@ -0,0 +1,152 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "feast-jc.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-jc.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "feast-jc.name" . }} + component: core + release: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.prometheus.enabled }} + prometheus.io/path: /metrics + prometheus.io/port: "{{ .Values.service.http.targetPort }}" + prometheus.io/scrape: "true" + {{- end }} + labels: + app: {{ template "feast-jc.name" . }} + component: core + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: {{ template "feast-jc.fullname" . }}-config + configMap: + name: {{ template "feast-jc.fullname" . }} + - name: {{ template "feast-jc.fullname" . }}-secret + secret: + secretName: {{ template "feast-jc.fullname" . }} + {{- if .Values.gcpServiceAccount.enabled }} + - name: {{ template "feast-jc.fullname" . }}-gcp-service-account + secret: + secretName: {{ .Values.gcpServiceAccount.existingSecret.name }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + + volumeMounts: + - name: {{ template "feast-jc.fullname" . }}-config + mountPath: /etc/feast + - name: {{ template "feast-jc.fullname" . }}-secret + mountPath: /etc/secrets/feast + readOnly: true + {{- if .Values.gcpServiceAccount.enabled }} + - name: {{ template "feast-jc.fullname" . }}-gcp-service-account + mountPath: /etc/secrets/google + readOnly: true + {{- end }} + + env: + - name: LOG_TYPE + value: {{ .Values.logType | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + + {{- if .Values.postgresql.existingSecret }} + - name: SPRING_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.existingSecret }} + key: postgresql-password + {{- end }} + + {{- if .Values.gcpServiceAccount.enabled }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/secrets/google/{{ .Values.gcpServiceAccount.existingSecret.key }} + {{- end }} + + {{- if .Values.gcpProjectId }} + - name: GOOGLE_CLOUD_PROJECT + value: {{ .Values.gcpProjectId | quote }} + {{- end }} + + {{- if .Values.javaOpts }} + - name: JAVA_TOOL_OPTIONS + value: {{ .Values.javaOpts }} + {{- end }} + + {{- range $key, $value := .Values.envOverrides }} + - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} + value: {{ $value | quote }} + {{- end }} + + command: + - java + - -jar + - /opt/feast/feast-job-coordinator.jar + - --spring.config.location= + {{- if index .Values "application.yaml" "enabled" -}} + classpath:/application.yml + {{- end }} + {{- if index .Values "application-generated.yaml" "enabled" -}} + ,file:/etc/feast/application-generated.yaml + {{- end }} + {{- if index .Values "application-secret.yaml" "enabled" -}} + ,file:/etc/secrets/feast/application-secret.yaml + {{- end }} + {{- if index .Values "application-override.yaml" "enabled" -}} + ,file:/etc/feast/application-override.yaml + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.http.targetPort }} + - name: grpc + containerPort: {{ .Values.service.grpc.targetPort }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 10 }} diff --git a/infra/charts/feast/charts/feast-jc/templates/ingress.yaml b/infra/charts/feast/charts/feast-jc/templates/ingress.yaml new file mode 100644 index 00000000000..7f453e1a75f --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/ingress.yaml @@ -0,0 +1,7 @@ +{{- if .Values.ingress.http.enabled -}} +{{ template "feast.ingress" (list . "core" "http" .Values.ingress.http) }} +{{- end }} +--- +{{ if .Values.ingress.grpc.enabled -}} +{{ template "feast.ingress" (list . "core" "grpc" .Values.ingress.grpc) }} +{{- end }} diff --git a/infra/charts/feast/charts/feast-jc/templates/secret.yaml b/infra/charts/feast/charts/feast-jc/templates/secret.yaml new file mode 100644 index 00000000000..d10f8614558 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/secret.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "feast-jc.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-jc.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +stringData: + application-secret.yaml: | +{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }} diff --git a/infra/charts/feast/charts/feast-jc/templates/service.yaml b/infra/charts/feast/charts/feast-jc/templates/service.yaml new file mode 100644 index 00000000000..2b4c49027f4 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/templates/service.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "feast-jc.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-jc.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.service.annotations }} + annotations: + {{ toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.http.port }} + targetPort: {{ .Values.service.http.targetPort }} + {{- if .Values.service.http.nodePort }} + nodePort: {{ .Values.service.http.nodePort }} + {{- end }} + - name: grpc + port: {{ .Values.service.grpc.port }} + targetPort: {{ .Values.service.grpc.targetPort }} + {{- if .Values.service.grpc.nodePort }} + nodePort: {{ .Values.service.grpc.nodePort }} + {{- end }} + selector: + app: {{ template "feast-jc.name" . }} + component: core + release: {{ .Release.Name }} + diff --git a/infra/charts/feast/charts/feast-jc/values.yaml b/infra/charts/feast/charts/feast-jc/values.yaml new file mode 100644 index 00000000000..806de28e294 --- /dev/null +++ b/infra/charts/feast/charts/feast-jc/values.yaml @@ -0,0 +1,156 @@ +# replicaCount -- Number of pods that will be created +replicaCount: 1 + +image: + # image.repository -- Docker image repository + repository: gcr.io/kf-feast/feast-jc + # image.tag -- Image tag + tag: 0.6.2 + # image.pullPolicy -- Image pull policy + pullPolicy: IfNotPresent + +application.yaml: + # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. + enabled: true + +application-generated.yaml: + # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. + enabled: true + +# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. +application-secret.yaml: + enabled: true + +# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` +application-override.yaml: + enabled: true + +gcpServiceAccount: + # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key + enabled: false + existingSecret: + # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account + name: feast-gcp-service-account + # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account) + key: credentials.json + +postgresql: + # postgresql.existingSecret -- Existing secret to use for authenticating to Postgres + existingSecret: "" + +# gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow +gcpProjectId: "" + +# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` +javaOpts: + +# logType -- Log format, either `JSON` or `Console` +logType: Console +# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` +logLevel: WARN + +prometheus: + # prometheus.enabled -- Flag to enable scraping of Feast Core metrics + enabled: true + +# By default we disable the liveness probe, since if the DB fails restarting core will not result +# in application healing. +livenessProbe: + # livenessProbe.enabled -- Flag to enabled the probe + enabled: false + # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 60 + # livenessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # livenessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 5 + # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +readinessProbe: + # readinessProbe.enabled -- Flag to enabled the probe + enabled: true + # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated + initialDelaySeconds: 20 + # readinessProbe.periodSeconds -- How often to perform the probe + periodSeconds: 10 + # readinessProbe.timeoutSeconds -- When the probe times out + timeoutSeconds: 10 + # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful + successThreshold: 1 + # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed + failureThreshold: 5 + +service: + # service.type -- Kubernetes service type + type: ClusterIP + http: + # service.http.port -- Service port for HTTP requests + port: 80 + # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics + targetPort: 8080 + # service.http.nodePort -- Port number that each cluster node will listen to + nodePort: + grpc: + # service.grpc.port -- Service port for GRPC requests + port: 6565 + # service.grpc.targetPort -- Container port serving GRPC requests + targetPort: 6565 + # service.grpc.nodePort -- Port number that each cluster node will listen to + nodePort: + +ingress: + grpc: + # ingress.grpc.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.grpc.class -- Which ingress controller to use + class: nginx + # ingress.grpc.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.grpc.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.grpc.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.grpc.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.grpc.auth.enabled -- Flag to enable auth + enabled: false + http: + # ingress.http.enabled -- Flag to create an ingress resource for the service + enabled: false + # ingress.http.class -- Which ingress controller to use + class: nginx + # ingress.http.hosts -- List of hostnames to match when routing requests + hosts: [] + # ingress.http.annotations -- Extra annotations for the ingress + annotations: {} + https: + # ingress.http.https.enabled -- Flag to enable HTTPS + enabled: true + # ingress.http.https.secretNames -- Map of hostname to TLS secret name + secretNames: {} + # ingress.http.whitelist -- Allowed client IP source ranges + whitelist: "" + auth: + # ingress.http.auth.enabled -- Flag to enable auth + enabled: false + # ingress.http.auth.authUrl -- URL to an existing authentication service + authUrl: http://auth-server.auth-ns.svc.cluster.local/auth + +# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) +resources: {} + +# nodeSelector -- Node labels for pod assignment +nodeSelector: {} + +# envOverrides -- Extra environment variables to set +envOverrides: {} + +# podLabels -- Labels to be added to Feast Core pods +podLabels: {} diff --git a/infra/charts/feast/values-dataflow-runner.yaml b/infra/charts/feast/values-dataflow-runner.yaml index 56e51551970..d5a290e362c 100644 --- a/infra/charts/feast/values-dataflow-runner.yaml +++ b/infra/charts/feast/values-dataflow-runner.yaml @@ -4,6 +4,15 @@ feast-core: enabled: true postgresql: existingSecret: feast-postgresql + application-override.yaml: + feast: + stream: + options: + bootstrapServers: + +feast-jc: + gcpServiceAccount: + enabled: true application-override.yaml: feast: stream: diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index 110d0312389..94a4b91cb28 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -2,6 +2,10 @@ feast-core: # feast-core.enabled -- Flag to install Feast Core enabled: true +feast-jc: + # feast-jc.enabled -- Flag to install Feast Job Coordinator + enabled: true + feast-online-serving: # feast-online-serving.enabled -- Flag to install Feast Online Serving enabled: true diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile index 6774a93d0bf..f19eb89eae1 100644 --- a/infra/docker/core/Dockerfile +++ b/infra/docker/core/Dockerfile @@ -31,18 +31,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . ARG REVISION=dev -RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \ +RUN mvn --also-make --projects core -Drevision=$REVISION \ -DskipUTs=true --batch-mode clean package -# -# Unpack the jar and copy the files into production Docker image -# for faster startup time when starting Dataflow jobs from Feast Core. -# This is because we need to stage the classes and dependencies when using Dataflow. -# The final size of the production image will be bigger but it seems -# a good tradeoff between speed and size. -# -# https://github.com/feast-dev/feast/pull/291 -RUN apt-get -qq update && apt-get -y install unar && \ - unar /build/core/target/feast-core-$REVISION-exec.jar -o /build/core/target/ # # Download grpc_health_probe to run health check for Feast Serving @@ -60,8 +50,6 @@ FROM openjdk:11-jre as production ARG REVISION=dev COPY --from=builder /build/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar -# Required for staging jar dependencies when submitting Dataflow jobs. -COPY --from=builder /build/core/target/feast-core-$REVISION-exec /opt/feast/feast-core COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ diff --git a/infra/docker/jc/Dockerfile b/infra/docker/jc/Dockerfile new file mode 100644 index 00000000000..683b4ab785c --- /dev/null +++ b/infra/docker/jc/Dockerfile @@ -0,0 +1,72 @@ +# ============================================================ +# Build stage 1: Builder +# ============================================================ + +FROM maven:3.6-jdk-11 as builder + +WORKDIR /build + +COPY pom.xml . +COPY datatypes/java/pom.xml datatypes/java/pom.xml +COPY common/pom.xml common/pom.xml +COPY auth/pom.xml auth/pom.xml +COPY ingestion/pom.xml ingestion/pom.xml +COPY core/pom.xml core/pom.xml +COPY serving/pom.xml serving/pom.xml +COPY storage/api/pom.xml storage/api/pom.xml +COPY storage/connectors/pom.xml storage/connectors/pom.xml +COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml +COPY storage/connectors/bigquery/pom.xml storage/connectors/bigquery/pom.xml +COPY sdk/java/pom.xml sdk/java/pom.xml +COPY job-coordinator/pom.xml job-coordinator/pom.xml +COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml +COPY protos/ protos/ + +# Setting Maven repository .m2 directory relative to /build folder gives the +# user to optionally use cached repository when building the image by copying +# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 +ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" +COPY pom.xml .m2/* .m2/ +RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true + +COPY . . + +ARG REVISION=dev +RUN mvn --also-make --projects job-coordinator,ingestion -Drevision=$REVISION \ + -DskipUTs=true --batch-mode clean package +# +# Unpack the jar and copy the files into production Docker image +# for faster startup time when starting Dataflow jobs from Feast Core. +# This is because we need to stage the classes and dependencies when using Dataflow. +# The final size of the production image will be bigger but it seems +# a good tradeoff between speed and size. +# +# https://github.com/feast-dev/feast/pull/291 +RUN apt-get -qq update && apt-get -y install unar && \ + unar /build/core/target/feast-job-coordinator-$REVISION-exec.jar -o /build/core/target/ + +# +# Download grpc_health_probe to run health check for Feast Serving +# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ +# +RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ + -O /usr/bin/grpc-health-probe && \ + chmod +x /usr/bin/grpc-health-probe + +# ============================================================ +# Build stage 2: Production +# ============================================================ + +FROM openjdk:11-jre as production +ARG REVISION=dev + +COPY --from=builder /build/core/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar +# Required for staging jar dependencies when submitting Dataflow jobs. +COPY --from=builder /build/core/target/feast-job-coordinator-$REVISION-exec /opt/feast/feast-job-coordinator +COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe + +CMD ["java",\ + "-Xms2048m",\ + "-Xmx2048m",\ + "-jar",\ + "/opt/feast/feast-job-coordinator.jar"] diff --git a/infra/docker/jc/Dockerfile.debug b/infra/docker/jc/Dockerfile.debug new file mode 100644 index 00000000000..e69de29bb2d diff --git a/infra/docker/jc/Dockerfile.dev b/infra/docker/jc/Dockerfile.dev new file mode 100644 index 00000000000..294926fa391 --- /dev/null +++ b/infra/docker/jc/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM openjdk:11-jre +ARG REVISION=dev +ADD $PWD/core/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar +CMD ["java",\ + "-Xms2048m",\ + "-Xmx2048m",\ + "-jar",\ + "/opt/feast/feast-job-coordinator.jar"] diff --git a/infra/scripts/setup-common-functions.sh b/infra/scripts/setup-common-functions.sh index 221294a60de..6c5a45cf27e 100755 --- a/infra/scripts/setup-common-functions.sh +++ b/infra/scripts/setup-common-functions.sh @@ -93,6 +93,21 @@ start_feast_core() { nc -w2 localhost 6565 /var/log/feast-jc.log & + ${SCRIPTS_DIR}/wait-for-it.sh localhost:6570 --timeout=90 + + tail -n10 /var/log/feast-jc.log + nc -w2 localhost 6570 /tmp/core.warehouse.application.yml +cat < /tmp/jc.warehouse.application.yml feast: + core-host: localhost + core-port: 6565 jobs: polling_interval_milliseconds: 10000 active_runner: direct @@ -71,7 +73,8 @@ feast: EOF -start_feast_core /tmp/core.warehouse.application.yml +start_feast_core +start_feast_jc /tmp/jc.warehouse.application.yml DATASET_NAME=feast_$(date +%s) bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \ diff --git a/infra/scripts/test-end-to-end-redis-cluster.sh b/infra/scripts/test-end-to-end-redis-cluster.sh index 9a989e51970..2df6f6b8bf1 100755 --- a/infra/scripts/test-end-to-end-redis-cluster.sh +++ b/infra/scripts/test-end-to-end-redis-cluster.sh @@ -41,8 +41,10 @@ else fi # Start Feast Core with auth if enabled -cat < /tmp/core.warehouse.application.yml +cat < /tmp/jc.warehouse.application.yml feast: + core-host: localhost + core-port: 6565 jobs: polling_interval_milliseconds: 5000 active_runner: direct @@ -52,7 +54,8 @@ feast: options: {} EOF -start_feast_core /tmp/core.warehouse.application.yml +start_feast_core +start_feast_jc /tmp/jc.warehouse.application.yml cat < /tmp/serving.online.application.yml feast: diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 2f4f1a17dbd..a759518c1f6 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -44,15 +44,8 @@ else fi # Start Feast Core with auth if enabled -cat < /tmp/auth.core.warehouse.application.yml +cat < /tmp/core.warehouse.application.yml feast: - jobs: - polling_interval_milliseconds: 5000 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: {} security: authentication: enabled: true @@ -64,8 +57,10 @@ feast: provider: none EOF -cat < /tmp/core.warehouse.application.yml +cat < /tmp/jc.warehouse.application.yml feast: + core-host: localhost + core-port: 6565 jobs: polling_interval_milliseconds: 5000 active_runner: direct @@ -106,11 +101,11 @@ if [[ ${ENABLE_AUTH} = "true" ]]; print_banner "Starting Feast Serving with auth" else print_banner "Starting Feast core without auth" - start_feast_core /tmp/core.warehouse.application.yml + start_feast_core print_banner "Starting Feast Serving without auth" fi - +start_feast_jc /tmp/jc.warehouse.application.yml start_feast_serving /tmp/serving.warehouse.application.yml install_python_with_miniconda_and_feast_sdk diff --git a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml index dd2bb6761bb..d47e7e8db9a 100644 --- a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml +++ b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml @@ -10,6 +10,20 @@ feast-core: image: tag: $IMAGE_TAG logLevel: INFO + application-override.yaml: + feast: + stream: + options: + bootstrapServers: $feast_kafka_1_ip:31090 + +feast-jc: + enabled: true + gcpServiceAccount: + enabled: true + service: + type: LoadBalancer + image: + tag: $IMAGE_TAG application-override.yaml: feast: stream: diff --git a/job-coordinator/pom.xml b/job-coordinator/pom.xml index e70394bb472..e7e2ca16f88 100644 --- a/job-coordinator/pom.xml +++ b/job-coordinator/pom.xml @@ -81,11 +81,11 @@ feast-common ${project.version} - - dev.feast - feast-auth - ${project.version} - + + + + + dev.feast feast-core @@ -101,11 +101,11 @@ - - org.springframework.boot - spring-boot-devtools - true - + + + + + javax.inject @@ -232,5 +232,11 @@ test true + + + org.springframework.security + spring-security-config + ${spring.security.version} + diff --git a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java index 343700db31c..fe946ad743e 100644 --- a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java +++ b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java @@ -51,8 +51,9 @@ public InMemoryJobRepository(JobManager jobManager) { this.jobManager = jobManager; this.storage = new HashMap<>(); - this.storage = - this.jobManager.listRunningJobs().stream().collect(Collectors.toMap(Job::getId, j -> j)); + // this.storage = + // this.jobManager.listRunningJobs().stream().collect(Collectors.toMap(Job::getId, j -> + // j)); } /** diff --git a/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java b/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java index 9c71d396008..567170753fb 100644 --- a/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java +++ b/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java @@ -17,109 +17,91 @@ package feast.jc.grpc; import com.google.api.gax.rpc.InvalidArgumentException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.auth.service.AuthorizationService; import feast.common.interceptors.GrpcMessageInterceptor; -import feast.jc.config.FeastProperties; import feast.jc.service.JobService; -import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase; import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.FeatureSetProto.FeatureSet; +import feast.proto.core.JobCoordinatorServiceGrpc.JobCoordinatorServiceImplBase; import io.grpc.Status; -import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; +import java.util.NoSuchElementException; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.context.SecurityContextHolder; - -import java.util.List; -import java.util.NoSuchElementException; -import java.util.stream.Collectors; /** Implementation of the feast core GRPC service. */ @Slf4j @GrpcService(interceptors = {GrpcMessageInterceptor.class}) -public class CoreServiceImpl extends CoreServiceImplBase { - - private final FeastProperties feastProperties; +public class CoreServiceImpl extends JobCoordinatorServiceImplBase { private JobService jobService; @Autowired - public CoreServiceImpl( - JobService jobService, - FeastProperties feastProperties) { + public CoreServiceImpl(JobService jobService) { this.jobService = jobService; - this.feastProperties = feastProperties; } - @Override - public void listIngestionJobs( - ListIngestionJobsRequest request, - StreamObserver responseObserver) { - try { - ListIngestionJobsResponse response = this.jobService.listJobs(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (InvalidArgumentException e) { - log.error("Received an invalid request on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } + @Override + public void listIngestionJobs( + ListIngestionJobsRequest request, + StreamObserver responseObserver) { + try { + ListIngestionJobsResponse response = this.jobService.listJobs(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (InvalidArgumentException e) { + log.error("Received an invalid request on calling listIngestionJobs method:", e); + responseObserver.onError( + Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling listIngestionJobs method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } + } - @Override - public void restartIngestionJob( - RestartIngestionJobRequest request, - StreamObserver responseObserver) { - try { - RestartIngestionJobResponse response = this.jobService.restartJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); - responseObserver.onError( - - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } + @Override + public void restartIngestionJob( + RestartIngestionJobRequest request, + StreamObserver responseObserver) { + try { + RestartIngestionJobResponse response = this.jobService.restartJob(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (NoSuchElementException e) { + log.error( + "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); + responseObserver.onError( + Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + } catch (UnsupportedOperationException e) { + log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); + responseObserver.onError( + Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling restartIngestionJob method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } + } - @Override - public void stopIngestionJob( - StopIngestionJobRequest request, StreamObserver - responseObserver) { - try { - StopIngestionJobResponse response = this.jobService.stopJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); - responseObserver.onError( - - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } + @Override + public void stopIngestionJob( + StopIngestionJobRequest request, StreamObserver responseObserver) { + try { + StopIngestionJobResponse response = this.jobService.stopJob(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (NoSuchElementException e) { + log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); + responseObserver.onError( + Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); + } catch (UnsupportedOperationException e) { + log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); + responseObserver.onError( + Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); + } catch (Exception e) { + log.error("Unexpected exception on calling stopIngestionJob method:", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); } + } } diff --git a/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java b/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java index 850d25ef17d..8ab0db966d9 100644 --- a/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java +++ b/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java @@ -19,8 +19,6 @@ import com.google.common.base.Objects; import feast.common.models.FeatureSetReference; import feast.proto.core.FeatureSetProto.FeatureSetJobDeliveryStatus; -import lombok.Getter; -import lombok.Setter; /** * Data class that represents connection between {@link Job} and FeatureSet. For all FeatureSets @@ -28,13 +26,37 @@ * featureSetDeliveryStatuses map. FeatureSet is determined by {@link FeatureSetReference}. Stores * delivery status and latest delivered version. */ -@Getter -@Setter public class FeatureSetDeliveryStatus { - private FeatureSetReference featureSetReference; + private final FeatureSetReference featureSetReference; private FeatureSetJobDeliveryStatus deliveryStatus; private int deliveredVersion; + public FeatureSetDeliveryStatus(FeatureSetReference featureSetReference) { + this.featureSetReference = featureSetReference; + } + + public FeatureSetDeliveryStatus setDeliveryStatus(FeatureSetJobDeliveryStatus deliveryStatus) { + this.deliveryStatus = deliveryStatus; + return this; + } + + public FeatureSetDeliveryStatus setDeliveredVersion(int deliveredVersion) { + this.deliveredVersion = deliveredVersion; + return this; + } + + public FeatureSetJobDeliveryStatus getDeliveryStatus() { + return deliveryStatus; + } + + public FeatureSetReference getFeatureSetReference() { + return featureSetReference; + } + + public int getDeliveredVersion() { + return deliveredVersion; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/job-coordinator/src/main/java/feast/jc/model/Job.java b/job-coordinator/src/main/java/feast/jc/model/Job.java index c007ea33804..03fbb8d476e 100644 --- a/job-coordinator/src/main/java/feast/jc/model/Job.java +++ b/job-coordinator/src/main/java/feast/jc/model/Job.java @@ -122,10 +122,9 @@ public void addAllStores(Set stores) { public void addAllFeatureSets(Set featureSets) { for (FeatureSetProto.FeatureSet fs : featureSets) { - FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(); FeatureSetReference ref = FeatureSetReference.of(fs.getSpec().getProject(), fs.getSpec().getName()); - status.setFeatureSetReference(ref); + FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(ref); if (fs.getMeta().getStatus() == FeatureSetProto.FeatureSetStatus.STATUS_READY) { // Feature Set was already delivered to previous generation of the job diff --git a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java index 6d64c1c9ffd..b50f628389b 100644 --- a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java +++ b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java @@ -39,6 +39,7 @@ import feast.proto.core.FeatureSetProto.FeatureSetSpec; import feast.proto.core.SourceProto.Source; import feast.proto.core.StoreProto.Store; +import io.grpc.StatusRuntimeException; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; @@ -93,7 +94,11 @@ public JobCoordinatorService( this.whitelistedStores = feastProperties.getJobs().getCoordinator().getWhitelistedStores(); this.currentVersion = feastProperties.getVersion(); this.jobLabels = new HashMap<>(feastProperties.getJobs().getCoordinator().getJobSelector()); - this.jobLabels.put(VERSION_LABEL, this.currentVersion); + this.jobLabels.put(VERSION_LABEL, getCurrentFeastVersion()); + } + + private String getCurrentFeastVersion() { + return this.currentVersion.replace(".", "-"); } /** @@ -234,7 +239,7 @@ private boolean jobRequiresUpgrade(Job job, Set stores) { return true; } - if (!this.currentVersion.equals(job.getLabels().get(VERSION_LABEL))) { + if (!getCurrentFeastVersion().equals(job.getLabels().get(VERSION_LABEL))) { return true; } @@ -275,8 +280,7 @@ FeatureSet allocateFeatureSetToJobs(FeatureSet featureSet) { continue; } - FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(); - status.setFeatureSetReference(ref); + FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(ref); status.setDeliveryStatus(FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_IN_PROGRESS); status.setDeliveredVersion(0); @@ -309,11 +313,18 @@ private Collection getExtraJobs(List keepJobs) { } private List getAllStores() { - ListStoresResponse listStoresResponse = - specService.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter(Filter.newBuilder().build()) - .build()); + ListStoresResponse listStoresResponse; + try { + listStoresResponse = + specService.listStores( + CoreServiceProto.ListStoresRequest.newBuilder() + .setFilter(Filter.newBuilder().build()) + .build()); + } catch (StatusRuntimeException e) { + log.error("Core Service is unavailable"); + return Collections.emptyList(); + } + return listStoresResponse.getStoreList().stream() .filter(s -> this.whitelistedStores.contains(s.getName())) .collect(Collectors.toList()); @@ -373,18 +384,24 @@ List getFeatureSetsForStore(Store store) { @Scheduled(fixedDelayString = "${feast.stream.specsOptions.notifyIntervalMilliseconds}") public void notifyJobsWhenFeatureSetUpdated() { - List pendingFeatureSets = - specService - .listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject("*") - .setFeatureSetName("*") - .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_PENDING) - .build()) - .build()) - .getFeatureSetsList(); + List pendingFeatureSets; + try { + pendingFeatureSets = + specService + .listFeatureSets( + CoreServiceProto.ListFeatureSetsRequest.newBuilder() + .setFilter( + CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() + .setProject("*") + .setFeatureSetName("*") + .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_PENDING) + .build()) + .build()) + .getFeatureSetsList(); + } catch (StatusRuntimeException exc) { + log.error("Core Service is unavailable"); + return; + } pendingFeatureSets.stream() .map(this::allocateFeatureSetToJobs) @@ -467,14 +484,19 @@ public void listenAckFromJobs( ConsumerRecord record) { String setReference = record.key(); FeatureSetReference ref = FeatureSetReference.parse(setReference); - FeatureSet featureSet = - specService - .getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setProject(ref.getProjectName()) - .setName(ref.getFeatureSetName()) - .build()) - .getFeatureSet(); + FeatureSet featureSet; + try { + featureSet = + specService + .getFeatureSet( + CoreServiceProto.GetFeatureSetRequest.newBuilder() + .setProject(ref.getProjectName()) + .setName(ref.getFeatureSetName()) + .build()) + .getFeatureSet(); + } catch (StatusRuntimeException e) { + featureSet = null; + } if (featureSet == null) { log.warn( diff --git a/job-coordinator/src/main/resources/application.yml b/job-coordinator/src/main/resources/application.yml index 8d05afe4073..f72c1e89050 100644 --- a/job-coordinator/src/main/resources/application.yml +++ b/job-coordinator/src/main/resources/application.yml @@ -85,9 +85,9 @@ feast: name: "*" # Stores names that are enabled on current instance of JobManager whitelisted-stores: + - historical - online - online_cluster - - historical stream: # Feature stream type. Only kafka is supported. @@ -116,7 +116,7 @@ feast: grpc: server: # The port that Feast Core gRPC service listens on - port: 6567 + port: 6570 security: enabled: false certificateChain: server.crt diff --git a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java index 26fbdaf3f79..ad0d876f1fb 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java @@ -29,19 +29,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleAPIClient; +import feast.common.it.*; import feast.common.util.KafkaSerialization; import feast.jc.config.FeastProperties; import feast.jc.dao.JobRepository; import feast.jc.model.Job; import feast.jc.model.JobStatus; import feast.jc.runner.JobManager; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.StoreProto; +import feast.proto.core.*; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -62,12 +57,15 @@ import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.util.SocketUtils; @SpringBootTest( properties = { "feast.jobs.enabled=true", - "feast.jobs.polling_interval_milliseconds=10000", - "feast.stream.specsOptions.notifyIntervalMilliseconds=100", + "feast.jobs.polling_interval_milliseconds=1000", + "feast.stream.specsOptions.notifyIntervalMilliseconds=1000", "feast.jobs.coordinator.consolidate-jobs-per-source=true", "feast.jobs.coordinator.feature-set-selector[0].name=test", "feast.jobs.coordinator.feature-set-selector[0].project=default", @@ -76,8 +74,7 @@ "feast.version=1.0.0" }) public class JobCoordinatorIT extends BaseIT { - @Autowired private JobManager jobManager2; - private FakeJobManager jobManager; + @Autowired private FakeJobManager jobManager; @Autowired private JobRepository jobRepository; @@ -85,30 +82,46 @@ public class JobCoordinatorIT extends BaseIT { static CoreServiceGrpc.CoreServiceBlockingStub stub; static List specsMailbox = new ArrayList<>(); - static SimpleAPIClient apiClient; + static SimpleCoreClient coreApiClient; + static SimpleJCClient apiClient; + + static int corePort = SocketUtils.findAvailableTcpPort(); + static ExternalApp coreApp = + ExternalApp.builder() + .setSpringApplication(feast.core.CoreApplication.class) + .setName("it-core") + .setGRPCPort(corePort) + .setPostgreSQL(postgreSQLContainer) + .build(); + + @DynamicPropertySource + static void properties(DynamicPropertyRegistry registry) { + registry.add("feast.corePort", () -> corePort); + } @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - // StandardEnvironment env = new StandardEnvironment(); - // env.setDefaultProfiles("it-core"); - // new SpringApplicationBuilder(feast.core.CoreApplication.class) - // .environment(env) - // .properties( - // ImmutableMap.of( - // "spring.datasource.url", postgreSQLContainer.getJdbcUrl(), - // "spring.datasource.username", postgreSQLContainer.getUsername(), - // "spring.datasource.password", postgreSQLContainer.getPassword())) - // .run(); + public static void globalSetUp(@Value("${grpc.server.port}") int localPort) { + coreApp.start(); ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); + ManagedChannelBuilder.forAddress("localhost", corePort).usePlaintext().build(); stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleAPIClient(stub); + coreApiClient = new SimpleCoreClient(stub); + + apiClient = + new SimpleJCClient( + JobCoordinatorServiceGrpc.newBlockingStub( + ManagedChannelBuilder.forAddress("localhost", localPort).usePlaintext().build())); + } + + @AfterAll + public static void globalTearDown() { + coreApp.stop(); } @BeforeEach public void setUp(TestInfo testInfo) { - apiClient.updateStore(DataGenerator.getDefaultStore()); + coreApiClient.updateStore(DataGenerator.getDefaultStore()); specsMailbox = new ArrayList<>(); @@ -131,10 +144,10 @@ public void listenSpecs(ConsumerRecord record) @Test @SneakyThrows public void shouldCreateJobForNewSource() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - List featureSets = apiClient.simpleListFeatureSets("*"); + List featureSets = coreApiClient.simpleListFeatureSets("*"); assertThat(featureSets.size(), equalTo(1)); await() @@ -156,12 +169,12 @@ public void shouldCreateJobForNewSource() { @Test public void shouldUpgradeJobWhenStoreChanged() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); await().until(jobManager::getAllJobs, hasSize(1)); - apiClient.updateStore( + coreApiClient.updateStore( DataGenerator.createStore( "new-store", StoreProto.Store.StoreType.REDIS, @@ -180,10 +193,10 @@ public void shouldUpgradeJobWhenStoreChanged() { @Test public void shouldRestoreJobThatStopped() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - await().until(jobManager::getAllJobs, hasSize(1)); + await().until(() -> jobRepository.findByStatus(JobStatus.RUNNING), hasSize(1)); Job job = jobRepository.findByStatus(JobStatus.RUNNING).get(0); List ingestionJobs = apiClient.listIngestionJobs(); @@ -206,7 +219,7 @@ public void shouldRestoreJobThatStopped() { @Test @SneakyThrows public void shouldNotCreateJobForUnwantedFeatureSet() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "other")); Thread.sleep(2000); @@ -217,7 +230,7 @@ public void shouldNotCreateJobForUnwantedFeatureSet() { @Test @SneakyThrows public void shouldRestartJobWithOldVersion() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); Job job = @@ -227,7 +240,7 @@ public void shouldRestartJobWithOldVersion() { ImmutableMap.of( DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) .setId("some-running-id") - .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "0.9.9")) + .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "0-9-9")) .build(); jobManager.startJob(job); @@ -238,7 +251,7 @@ public void shouldRestartJobWithOldVersion() { Job replacement = jobRepository.findByStatus(JobStatus.RUNNING).get(0); assertThat(replacement.getSource(), equalTo(job.getSource())); assertThat(replacement.getStores(), equalTo(job.getStores())); - assertThat(replacement.getLabels(), hasEntry(JobCoordinatorService.VERSION_LABEL, "1.0.0")); + assertThat(replacement.getLabels(), hasEntry(JobCoordinatorService.VERSION_LABEL, "1-0-0")); } @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -265,13 +278,13 @@ public void shouldSendNewSpec() { ImmutableMap.of( DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) .setId("some-running-id") - .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "1.0.0")) + .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "1-0-0")) .build(); jobManager.startJob(job); jobRepository.add(job); - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet( DataGenerator.getDefaultSource(), "default", @@ -279,7 +292,7 @@ public void shouldSendNewSpec() { ImmutableMap.of("entity", ValueProto.ValueType.Enum.BOOL), ImmutableMap.of())); - FeatureSetProto.FeatureSet featureSet = apiClient.simpleGetFeatureSet("default", "test"); + FeatureSetProto.FeatureSet featureSet = coreApiClient.simpleGetFeatureSet("default", "test"); assertThat( featureSet.getMeta().getStatus(), @@ -301,7 +314,7 @@ public void shouldSendNewSpec() { @Test @Order(2) public void shouldUpdateSpec() { - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet( DataGenerator.getDefaultSource(), "default", @@ -334,7 +347,7 @@ public void shouldIgnoreOutdatedACKs() throws InterruptedException { // time to process Thread.sleep(1000); - FeatureSetProto.FeatureSet featureSet = apiClient.simpleGetFeatureSet("default", "test"); + FeatureSetProto.FeatureSet featureSet = coreApiClient.simpleGetFeatureSet("default", "test"); assertThat( featureSet.getMeta().getStatus(), @@ -354,7 +367,7 @@ public void shouldUpdateDeliveryStatus() { await() .until( - () -> apiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), + () -> coreApiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), equalTo(FeatureSetProto.FeatureSetStatus.STATUS_READY)); } @@ -363,7 +376,7 @@ public void shouldUpdateDeliveryStatus() { public void shouldReallocateFeatureSetAfterSourceChanged() { assertThat(jobManager.getJobStatus(job), equalTo(JobStatus.RUNNING)); - apiClient.simpleApplyFeatureSet( + coreApiClient.simpleApplyFeatureSet( DataGenerator.createFeatureSet( DataGenerator.createSource("localhost", "newTopic"), "default", @@ -398,13 +411,13 @@ public void shouldUpdateStatusAfterACKFromNewJob() { await() .until( - () -> apiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), + () -> coreApiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), equalTo(FeatureSetProto.FeatureSetStatus.STATUS_READY)); } } @TestConfiguration - public static class TestConfig { + public static class TestConfig extends BaseIT.BaseTestConfig { @Bean public JobManager getJobManager() { return new FakeJobManager(); diff --git a/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java b/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java index ed20cb0cac3..10c402fc115 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java @@ -31,9 +31,9 @@ import feast.jc.model.Job; import feast.jc.model.JobStatus; import feast.jc.runner.JobManager; -import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceProto; import feast.proto.core.FeatureSetReferenceProto; +import feast.proto.core.JobCoordinatorServiceGrpc; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; @@ -50,7 +50,7 @@ public class JobServiceIT extends BaseIT { @Autowired private JobRepository jobRepository; - static CoreServiceGrpc.CoreServiceBlockingStub stub; + static JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub; Job job; @@ -58,7 +58,7 @@ public class JobServiceIT extends BaseIT { public static void globalSetUp(@Value("${grpc.server.port}") int port) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); + stub = JobCoordinatorServiceGrpc.newBlockingStub(channel); } private Job createJobWithId(String jobId) { @@ -127,7 +127,7 @@ private void assertReturnsJob(CoreServiceProto.ListIngestionJobsRequest.Filter f @Test public void shouldReturnListOfJobsByByFeatureSetReference() { FeatureSetReference ref = FeatureSetReference.of("default", "fs"); - FeatureSetDeliveryStatus featureSetDeliveryStatus = new FeatureSetDeliveryStatus(); + FeatureSetDeliveryStatus featureSetDeliveryStatus = new FeatureSetDeliveryStatus(ref); this.job.getFeatureSetDeliveryStatuses().put(ref, featureSetDeliveryStatus); diff --git a/job-coordinator/src/test/resources/application-it-core.yml b/job-coordinator/src/test/resources/application-it-core.yml index 9ad4e7037cd..6072acd39e5 100644 --- a/job-coordinator/src/test/resources/application-it-core.yml +++ b/job-coordinator/src/test/resources/application-it-core.yml @@ -39,8 +39,6 @@ feast: grpc: server: - # The port that Feast Core gRPC service listens on - port: 6565 security: enabled: false diff --git a/pom.xml b/pom.xml index 8bb56a9fe55..56d65ec708a 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 3.12.2 3.12.2 2.3.1.RELEASE - 5.2.5.RELEASE + 5.2.7.RELEASE 5.3.0.RELEASE 2.9.0.RELEASE 2.22.0 diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index e9cfff4294b..debaf6f13c8 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -22,8 +22,8 @@ import yaml from feast.client import Client -from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient from feast.config import Config +from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.feature_set import FeatureSet, FeatureSetRef from feast.loaders.yaml import yaml_loader @@ -422,7 +422,7 @@ def ingest_job_restart(job_id: str): Waits for the job to fully restart. """ # find ingestion job for id - feast_client = Client() + feast_client = JobCoordinatorClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") diff --git a/sdk/python/feast/contrib/job_coordinator/client.py b/sdk/python/feast/contrib/job_coordinator/client.py index 774b5c43ca7..e03fd2435e1 100644 --- a/sdk/python/feast/contrib/job_coordinator/client.py +++ b/sdk/python/feast/contrib/job_coordinator/client.py @@ -3,18 +3,23 @@ import grpc from feast.config import Config -from feast.constants import CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, CONFIG_CORE_SERVER_SSL_CERT_KEY, \ - CONFIG_ENABLE_AUTH_KEY, CONFIG_CORE_ENABLE_SSL_KEY, CONFIG_JC_SERVER_KEY +from feast.constants import ( + CONFIG_CORE_ENABLE_SSL_KEY, + CONFIG_CORE_SERVER_SSL_CERT_KEY, + CONFIG_ENABLE_AUTH_KEY, + CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, + CONFIG_JC_SERVER_KEY, +) +from feast.contrib.job_coordinator.job import IngestJob from feast.core.CoreService_pb2 import ( ListIngestionJobsRequest, RestartIngestionJobRequest, - StopIngestionJobRequest + StopIngestionJobRequest, ) from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub from feast.feature_set import FeatureSetRef -from feast.grpc.grpc import create_grpc_channel from feast.grpc import auth as feast_auth -from feast.contrib.job_coordinator.job import IngestJob +from feast.grpc.grpc import create_grpc_channel class Client: @@ -53,7 +58,7 @@ def _jc_service(self): timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) self._jc_service_stub = JobCoordinatorServiceStub(channel) - + return self._jc_service_stub def list_ingest_jobs( @@ -137,4 +142,4 @@ def _get_grpc_metadata(self): """ if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY) and self._auth_metadata: return self._auth_metadata.get_signed_meta() - return () \ No newline at end of file + return () diff --git a/sdk/python/feast/contrib/job_coordinator/job.py b/sdk/python/feast/contrib/job_coordinator/job.py index 4c2faadcfbf..8f2c0738daa 100644 --- a/sdk/python/feast/contrib/job_coordinator/job.py +++ b/sdk/python/feast/contrib/job_coordinator/job.py @@ -6,7 +6,8 @@ from feast import Source from feast.core.CoreService_pb2 import ListIngestionJobsRequest from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub -from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto, IngestionJobStatus +from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto +from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.core.Store_pb2 import Store from feast.feature_set import FeatureSetRef from feast.wait import wait_retry_backoff @@ -118,4 +119,4 @@ def __str__(self): def __repr__(self): # render the ingest job as human readable string - return f"IngestJob<{self.id}>" \ No newline at end of file + return f"IngestJob<{self.id}>" diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index bca95a664b2..5f86e9cf779 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -29,6 +29,7 @@ from feast.client import Client from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient +from feast.contrib.job_coordinator.job import IngestJob from feast.core import CoreService_pb2_grpc as Core from feast.core.CoreService_pb2 import ( GetFeastCoreVersionResponse, @@ -50,7 +51,6 @@ from feast.entity import Entity from feast.feature import Feature from feast.feature_set import FeatureSet, FeatureSetRef -from feast.contrib.job_coordinator.job import IngestJob from feast.serving import ServingService_pb2_grpc as Serving from feast.serving.ServingService_pb2 import DataFormat, FeastServingType from feast.serving.ServingService_pb2 import FeatureReference as FeatureRefProto diff --git a/tests/e2e/bq/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py index 209e1405cc4..ac67beab0cf 100644 --- a/tests/e2e/bq/bq-batch-retrieval.py +++ b/tests/e2e/bq/bq-batch-retrieval.py @@ -18,6 +18,7 @@ from bq.testutils import assert_stats_equal, clear_unsupported_fields from feast.client import Client +from feast.contrib.job_coordinator.client import Client as JCClient from feast.core.CoreService_pb2 import ListStoresRequest from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.entity import Entity @@ -40,6 +41,11 @@ def serving_url(pytestconfig): return pytestconfig.getoption("serving_url") +@pytest.fixture(scope="module") +def jc_url(pytestconfig): + return pytestconfig.getoption("jc_url") + + @pytest.fixture(scope="module") def allow_dirty(pytestconfig): return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False @@ -497,9 +503,8 @@ def check(): @pytest.fixture(scope="module", autouse=True) -def infra_teardown(pytestconfig, core_url, serving_url): - client = Client(core_url=core_url, serving_url=serving_url) - client.set_project(PROJECT_NAME) +def infra_teardown(pytestconfig, jc_url): + client = JCClient(jc_url=jc_url) marker = pytestconfig.getoption("-m") yield marker diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index a38632a028a..72f8c07bdad 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -17,6 +17,7 @@ from feast.client import Client from feast.config import Config from feast.constants import CONFIG_AUTH_PROVIDER +from feast.contrib.job_coordinator.client import Client as JCClient from feast.core import CoreService_pb2 from feast.core.CoreService_pb2 import ApplyFeatureSetResponse, GetFeatureSetResponse from feast.core.CoreService_pb2_grpc import CoreServiceStub @@ -101,6 +102,11 @@ def serving_url(pytestconfig): return pytestconfig.getoption("serving_url") +@pytest.fixture(scope="module") +def jc_url(pytestconfig): + return pytestconfig.getoption("jc_url") + + @pytest.fixture(scope="module") def allow_dirty(pytestconfig): return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False @@ -140,6 +146,12 @@ def client(core_url, serving_url, allow_dirty, enable_auth): return client +@pytest.fixture(scope="module") +def jc_client(jc_url): + client = JCClient(jc_url=jc_url) + return client + + @pytest.fixture(scope="module") def ingest_time(): return datetime.utcnow() @@ -953,12 +965,12 @@ def try_get_features(): @pytest.mark.timeout(300) @pytest.mark.run(order=35) -def test_all_types_ingest_jobs(client, all_types_dataframe): +def test_all_types_ingest_jobs(jc_client, client, all_types_dataframe): # list ingestion jobs given featureset client.set_project(PROJECT_NAME) all_types_fs = client.get_feature_set(name="all_types") - ingest_jobs = client.list_ingest_jobs( + ingest_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs) ) # filter ingestion jobs to only those that are running @@ -971,14 +983,14 @@ def test_all_types_ingest_jobs(client, all_types_dataframe): # restart ingestion ingest_job # restart means stop current job # (replacement will be automatically spawned) - client.restart_ingest_job(ingest_job) + jc_client.restart_ingest_job(ingest_job) # wait for replacement to be created time.sleep(15) # should be more than polling_interval # id without timestamp part # that remains the same between jobs shared_id = "-".join(ingest_job.id.split("-")[:-1]) - ingest_jobs = client.list_ingest_jobs( + ingest_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs) ) replacement_jobs = [ @@ -996,7 +1008,7 @@ def test_all_types_ingest_jobs(client, all_types_dataframe): assert replacement_job.status == IngestionJobStatus.RUNNING # stop ingestion ingest_job - client.stop_ingest_job(replacement_job) + jc_client.stop_ingest_job(replacement_job) replacement_job.wait(IngestionJobStatus.ABORTED) assert replacement_job.status == IngestionJobStatus.ABORTED @@ -1255,7 +1267,7 @@ def test_list_entities_and_features(client): @pytest.mark.timeout(500) @pytest.mark.run(order=70) -def test_sources_deduplicate_ingest_jobs(client, kafka_brokers): +def test_sources_deduplicate_ingest_jobs(client, jc_client, kafka_brokers): shared_source = KafkaSource(kafka_brokers, "dup_shared") dup_source_fs_1 = FeatureSet( name="duplicate_source_fs_1", @@ -1267,12 +1279,12 @@ def test_sources_deduplicate_ingest_jobs(client, kafka_brokers): dup_source_fs_2.name = "duplicate_source_fs_2" def is_same_jobs(): - fs_1_jobs = client.list_ingest_jobs( + fs_1_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_1.name, project=dup_source_fs_1.project ) ) - fs_2_jobs = client.list_ingest_jobs( + fs_2_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_2.name, project=dup_source_fs_2.project ) @@ -1292,12 +1304,12 @@ def is_same_jobs(): return same def is_different_jobs(): - fs_1_jobs = client.list_ingest_jobs( + fs_1_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_1.name, project=dup_source_fs_1.project ) ) - fs_2_jobs = client.list_ingest_jobs( + fs_2_jobs = jc_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_2.name, project=dup_source_fs_2.project ) From 930e04d1de1f33e8c71081fd1650822a56276947 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 12:51:11 +0800 Subject: [PATCH 05/26] fix core rest it --- .../core/controller/CoreServiceRestIT.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java index f59eba7d503..7abdb5510d8 100644 --- a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java +++ b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java @@ -22,9 +22,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.ImmutableMap; -import feast.core.it.BaseIT; -import feast.core.it.DataGenerator; -import feast.core.it.SimpleAPIClient; +import feast.common.it.BaseIT; +import feast.common.it.DataGenerator; +import feast.common.it.SimpleCoreClient; import feast.core.model.Project; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.FeatureSetProto.FeatureSet; @@ -50,9 +50,20 @@ public class CoreServiceRestIT extends BaseIT { static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleAPIClient apiClient; + static SimpleCoreClient apiClient; @LocalServerPort private int port; + @TestConfiguration + public static class TestConfig extends BaseTestConfig {} + + @BeforeAll + public static void globalSetUp(@Value("${grpc.server.port}") int port) { + ManagedChannel channel = + ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); + stub = CoreServiceGrpc.newBlockingStub(channel); + apiClient = new SimpleCoreClient(stub); + } + @Test public void getVersion() { String uriString = UriComponentsBuilder.fromPath("/api/v1/version").toUriString(); @@ -194,17 +205,6 @@ public void listFeatures() { .body("features", aMapWithSize(2)); } - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleAPIClient(stub); - } - @BeforeEach private void createFakeFeatureSets() { // spec: From 5705ca1896941d4757f30b37a36885cdbf0932d9 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 13:30:56 +0800 Subject: [PATCH 06/26] revert bq changes --- .github/workflows/complete.yml | 2 +- .github/workflows/master_only.yml | 2 +- .../src/main/java/feast/common/it/BaseIT.java | 2 +- .../connectors/bigquery/writer/BigQueryWrite.java | 15 +++++++++++---- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 9300227bf51..7565ba70f16 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -11,7 +11,7 @@ jobs: env: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} REGISTRY: gcr.io/kf-feast - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar + MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar steps: - uses: actions/checkout@v2 - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 424b670f601..e144e23fb78 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -13,7 +13,7 @@ jobs: matrix: component: [core, serving, jc, jupyter, ci] env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar + MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar steps: - uses: actions/checkout@v2 - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master diff --git a/common-test/src/main/java/feast/common/it/BaseIT.java b/common-test/src/main/java/feast/common/it/BaseIT.java index c6001f209cc..f82a804ee1c 100644 --- a/common-test/src/main/java/feast/common/it/BaseIT.java +++ b/common-test/src/main/java/feast/common/it/BaseIT.java @@ -105,7 +105,7 @@ public ConsumerFactory testConsumerFactory() { Map props = new HashMap<>(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "test"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, this.getClass().getName()); return new DefaultKafkaConsumerFactory<>( props, new StringDeserializer(), new ByteArrayDeserializer()); diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java index 3a07092fbcc..3ebe57fce71 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java @@ -24,6 +24,8 @@ import feast.proto.types.FeatureRowProto.FeatureRow; import feast.storage.api.writer.FailedElement; import feast.storage.api.writer.WriteResult; +import feast.storage.connectors.bigquery.compression.CompactFeatureRows; +import feast.storage.connectors.bigquery.compression.FeatureRowsBatch; import java.util.List; import java.util.Map; import org.apache.beam.sdk.coders.KvCoder; @@ -179,10 +181,10 @@ public void processElement(ProcessContext context) {} private PCollection mergeInputWithResult( PCollection inputInFixedWindow, PCollection> successful) { - final TupleTag> inputTag = new TupleTag<>(); + final TupleTag inputTag = new TupleTag<>(); final TupleTag successTag = new TupleTag<>(); - PCollection>> insertedRows = + PCollection> insertedRows = inputInFixedWindow .apply( "MakeElementKey", @@ -199,7 +201,7 @@ public void process(ProcessContext c) { element)); } })) - .apply(Sample.fixedSizePerKey(compactionBatchSize)); + .apply(new CompactFeatureRows(compactionBatchSize)); PCollection> inputWithResult = KeyedPCollectionTuple.of(inputTag, insertedRows) @@ -229,7 +231,12 @@ public void process(ProcessContext c) { return; } - result.getAll(inputTag).forEach(rows -> rows.forEach(c::output)); + result + .getAll(inputTag) + .forEach( + rows -> + rows.getFeatureRowsSample(maxSuccessfulOutputs) + .forEachRemaining(c::output)); } })); } From ad8f0b40c6db60e6eba860aa9a3072d0eabfbe02 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 13:41:57 +0800 Subject: [PATCH 07/26] fix path --- infra/docker/jc/Dockerfile | 2 +- infra/scripts/setup-common-functions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/docker/jc/Dockerfile b/infra/docker/jc/Dockerfile index 683b4ab785c..8d13a3d5bed 100644 --- a/infra/docker/jc/Dockerfile +++ b/infra/docker/jc/Dockerfile @@ -43,7 +43,7 @@ RUN mvn --also-make --projects job-coordinator,ingestion -Drevision=$REVISION \ # # https://github.com/feast-dev/feast/pull/291 RUN apt-get -qq update && apt-get -y install unar && \ - unar /build/core/target/feast-job-coordinator-$REVISION-exec.jar -o /build/core/target/ + unar /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar -o /build/core/target/ # # Download grpc_health_probe to run health check for Feast Serving diff --git a/infra/scripts/setup-common-functions.sh b/infra/scripts/setup-common-functions.sh index 6c5a45cf27e..8b3f4c6f595 100755 --- a/infra/scripts/setup-common-functions.sh +++ b/infra/scripts/setup-common-functions.sh @@ -101,7 +101,7 @@ start_feast_jc() { export CONFIG_ARG="--spring.config.location=classpath:/application.yml,file://$1" fi - nohup java -jar core/target/feast-job-coordinator-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-jc.log & + nohup java -jar job-coordinator/target/feast-job-coordinator-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-jc.log & ${SCRIPTS_DIR}/wait-for-it.sh localhost:6570 --timeout=90 tail -n10 /var/log/feast-jc.log From bb704f048990ba764840f110f9d434495307c3da Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 13:51:04 +0800 Subject: [PATCH 08/26] fix job coordinator it --- .../src/test/java/feast/jc/service/JobCoordinatorIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java index ad0d876f1fb..d367ec43652 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java +++ b/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java @@ -25,6 +25,7 @@ import static org.hamcrest.collection.IsMapContaining.hasEntry; import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; import static org.hamcrest.core.AllOf.allOf; +import static org.hamcrest.number.OrderingComparison.greaterThan; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -388,6 +389,8 @@ public void shouldReallocateFeatureSetAfterSourceChanged() { await().until(() -> jobRepository.findByStatus(JobStatus.RUNNING), hasSize(1)); + await().until(() -> specsMailbox, hasSize(greaterThan(0))); + assertThat( specsMailbox.get(0), allOf( From 83c08427e47a96ac8b4a35f341bba82b2ba4b092 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 13:58:31 +0800 Subject: [PATCH 09/26] fix path in docker --- infra/docker/jc/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/docker/jc/Dockerfile b/infra/docker/jc/Dockerfile index 8d13a3d5bed..2b10b9d3e42 100644 --- a/infra/docker/jc/Dockerfile +++ b/infra/docker/jc/Dockerfile @@ -36,14 +36,14 @@ RUN mvn --also-make --projects job-coordinator,ingestion -Drevision=$REVISION \ -DskipUTs=true --batch-mode clean package # # Unpack the jar and copy the files into production Docker image -# for faster startup time when starting Dataflow jobs from Feast Core. +# for faster startup time when starting Dataflow jobs from Feast JC. # This is because we need to stage the classes and dependencies when using Dataflow. # The final size of the production image will be bigger but it seems # a good tradeoff between speed and size. # # https://github.com/feast-dev/feast/pull/291 RUN apt-get -qq update && apt-get -y install unar && \ - unar /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar -o /build/core/target/ + unar /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar -o /build/job-coordinator/target/ # # Download grpc_health_probe to run health check for Feast Serving @@ -60,9 +60,9 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa FROM openjdk:11-jre as production ARG REVISION=dev -COPY --from=builder /build/core/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar +COPY --from=builder /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar # Required for staging jar dependencies when submitting Dataflow jobs. -COPY --from=builder /build/core/target/feast-job-coordinator-$REVISION-exec /opt/feast/feast-job-coordinator +COPY --from=builder /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec /opt/feast/feast-job-coordinator COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ From e83e0324d19d24852f6d136883404ae11c22fd0c Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 14:10:29 +0800 Subject: [PATCH 10/26] jc web port --- job-coordinator/src/main/resources/application.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/job-coordinator/src/main/resources/application.yml b/job-coordinator/src/main/resources/application.yml index f72c1e89050..ef15ddc6c00 100644 --- a/job-coordinator/src/main/resources/application.yml +++ b/job-coordinator/src/main/resources/application.yml @@ -131,3 +131,8 @@ management: enabled: true host: ${STATSD_HOST:localhost} port: ${STATSD_PORT:8125} + +server: + # The port number on which the Tomcat webserver that serves REST API endpoints should listen + # Set default value avoiding conflicts with core & serving + port: ${SERVER_PORT:8082} \ No newline at end of file From 41c93ce9ff14c4bd7755a8cd422c5ef5f547e581 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 14:45:28 +0800 Subject: [PATCH 11/26] bean validation from apache.bval --- infra/scripts/setup-common-functions.sh | 3 +- job-coordinator/pom.xml | 59 ++++++++----------------- 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/infra/scripts/setup-common-functions.sh b/infra/scripts/setup-common-functions.sh index 8b3f4c6f595..445a3c8cc51 100755 --- a/infra/scripts/setup-common-functions.sh +++ b/infra/scripts/setup-common-functions.sh @@ -68,7 +68,7 @@ install_and_start_local_zookeeper_and_kafka() { build_feast_core_and_serving() { print_banner "Building Feast Core and Feast Serving" infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2020-08-19.tar \ --output-dir /root/ # Build jars for Feast @@ -76,6 +76,7 @@ build_feast_core_and_serving() { ls -lh core/target/*jar ls -lh serving/target/*jar + ls -lh job-coordinator/target/*jar } start_feast_core() { diff --git a/job-coordinator/pom.xml b/job-coordinator/pom.xml index e7e2ca16f88..5cebdd95e1f 100644 --- a/job-coordinator/pom.xml +++ b/job-coordinator/pom.xml @@ -81,31 +81,6 @@ feast-common ${project.version} - - - - - - - dev.feast - feast-core - ${project.version} - test - - - dev.feast - feast-common-test - ${project.version} - test - - - - - - - - javax.inject @@ -206,6 +181,17 @@ 1.2.3-alpha + + javax.xml.bind + jaxb-api + + + + org.apache.bval + bval-jsr + 2.0.4 + + com.jayway.jsonpath @@ -213,30 +199,23 @@ 2.2.0 test - - javax.xml.bind - jaxb-api + dev.feast + feast-core + ${project.version} + test - - - com.google.auto.value - auto-value-annotations - 1.6.6 + dev.feast + feast-common-test + ${project.version} + test - org.postgresql postgresql test true - - - org.springframework.security - spring-security-config - ${spring.security.version} - From 48f7712395f89d3b677cb63a59b81e3232a3eb79 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 14:47:01 +0800 Subject: [PATCH 12/26] e2e auth --- infra/scripts/test-end-to-end.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index a759518c1f6..19f683a85b8 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -97,7 +97,7 @@ EOF if [[ ${ENABLE_AUTH} = "true" ]]; then print_banner "Starting Feast core with auth" - start_feast_core /tmp/auth.core.warehouse.application.yml + start_feast_core /tmp/core.warehouse.application.yml print_banner "Starting Feast Serving with auth" else print_banner "Starting Feast core without auth" From 9fd042c809121e78f3117a19444939375b1b5bdf Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 15:04:17 +0800 Subject: [PATCH 13/26] default jc_url --- tests/e2e/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 61140699f93..ab7e171c254 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,6 +1,7 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") parser.addoption("--serving_url", action="store", default="localhost:6566") + parser.addoption("--jc_url", action="store", default="localhost:6570") parser.addoption("--allow_dirty", action="store", default="False") parser.addoption( "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/" From cc4c6d4957a294bea3058339525e03e4883d2862 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 15:38:47 +0800 Subject: [PATCH 14/26] jc_ip in dataflow e2e --- infra/charts/feast/charts/feast-jc/values.yaml | 4 ++-- infra/scripts/test-end-to-end-batch-dataflow.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infra/charts/feast/charts/feast-jc/values.yaml b/infra/charts/feast/charts/feast-jc/values.yaml index 806de28e294..18ee7de9cf1 100644 --- a/infra/charts/feast/charts/feast-jc/values.yaml +++ b/infra/charts/feast/charts/feast-jc/values.yaml @@ -95,9 +95,9 @@ service: nodePort: grpc: # service.grpc.port -- Service port for GRPC requests - port: 6565 + port: 6570 # service.grpc.targetPort -- Container port serving GRPC requests - targetPort: 6565 + targetPort: 6570 # service.grpc.nodePort -- Port number that each cluster node will listen to nodePort: diff --git a/infra/scripts/test-end-to-end-batch-dataflow.sh b/infra/scripts/test-end-to-end-batch-dataflow.sh index d1e3f271523..83434dfb4ae 100755 --- a/infra/scripts/test-end-to-end-batch-dataflow.sh +++ b/infra/scripts/test-end-to-end-batch-dataflow.sh @@ -284,9 +284,11 @@ cd $ORIGINAL_DIR/tests/e2e core_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}" service ${HELM_RELEASE_NAME}-feast-core) serving_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}" service ${HELM_RELEASE_NAME}-feast-batch-serving) +jc_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}" service ${HELM_RELEASE_NAME}-feast-jc) set +e -pytest -s -v bq/bq-batch-retrieval.py -m dataflow_runner --core_url "$core_ip:6565" --serving_url "$serving_ip:6566" --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest -s -v bq/bq-batch-retrieval.py -m dataflow_runner --core_url "$core_ip:6565" --serving_url "$serving_ip:6566" \ + --jc_url "$jc_ip:6570" --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then From 6c01f7671d16259d1669f60c21db590be8aeed08 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 15:54:23 +0800 Subject: [PATCH 15/26] docker compose & health impl --- infra/docker-compose/.env.sample | 1 + infra/docker-compose/core/core.yml | 9 ---- infra/docker-compose/docker-compose.yml | 17 ++++++ infra/docker-compose/jc/jc.yml | 15 ++++++ infra/scripts/test-load.sh | 7 +++ .../feast/jc/dao/InMemoryJobRepository.java | 6 +-- .../java/feast/jc/grpc/HealthServiceImpl.java | 54 +++++++++++++++++++ 7 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 infra/docker-compose/jc/jc.yml create mode 100644 job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index 5c1f5af819b..feedd4a3262 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,6 +2,7 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=0.6.2 GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json FEAST_CORE_CONFIG=./core/core.yml +FEAST_JC_CONFIG=./jc/jc.yml FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml FEAST_HISTORICAL_SERVING_ENABLED=false FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml \ No newline at end of file diff --git a/infra/docker-compose/core/core.yml b/infra/docker-compose/core/core.yml index 7506a2dbe61..16bd3a81188 100644 --- a/infra/docker-compose/core/core.yml +++ b/infra/docker-compose/core/core.yml @@ -1,13 +1,4 @@ feast: - jobs: - polling_interval_milliseconds: 20000 - job_update_timeout_seconds: 240 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: - tempLocation: gs://bucket/tempLocation stream: type: kafka options: diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 61fc873f41f..4d5225b7e43 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -21,6 +21,23 @@ services: - /opt/feast/feast-core.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + jc: + image: gcr.io/kf-feast/feast-jc:${FEAST_VERSION} + volumes: + - ${FEAST_JC_CONFIG}:/etc/feast/application.yml + - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json + environment: + GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json + depends_on: + - kafka + ports: + - 6570:6570 + command: + - java + - -jar + - /opt/feast/feast-job-coordinator.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} volumes: diff --git a/infra/docker-compose/jc/jc.yml b/infra/docker-compose/jc/jc.yml new file mode 100644 index 00000000000..cb57ab074f6 --- /dev/null +++ b/infra/docker-compose/jc/jc.yml @@ -0,0 +1,15 @@ +feast: + jobs: + polling_interval_milliseconds: 20000 + job_update_timeout_seconds: 240 + active_runner: direct + runners: + - name: direct + type: DirectRunner + options: + tempLocation: gs://bucket/tempLocation + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: "kafka:9092,localhost:9094" \ No newline at end of file diff --git a/infra/scripts/test-load.sh b/infra/scripts/test-load.sh index 6ff6d86231a..88842654173 100755 --- a/infra/scripts/test-load.sh +++ b/infra/scripts/test-load.sh @@ -36,6 +36,7 @@ else fi wait_for_docker_image gcr.io/kf-feast/feast-core:"${FEAST_VERSION}" +wait_for_docker_image gcr.io/kf-feast/feast-jc:"${FEAST_VERSION}" wait_for_docker_image gcr.io/kf-feast/feast-serving:"${FEAST_VERSION}" wait_for_docker_image gcr.io/kf-feast/feast-jupyter:"${FEAST_VERSION}" @@ -65,6 +66,12 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett # Wait for Feast Core to be ready "${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120 +# Get Feast Job Coordinator container IP address +export FEAST_JC_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jc_1) + +# Wait for Feast Job Coordinator to be ready +"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JC_CONTAINER_IP_ADDRESS}:6570 --timeout=120 + # Get Feast Online Serving container IP address export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1) diff --git a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java index fe946ad743e..76e960a50b3 100644 --- a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java +++ b/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java @@ -49,11 +49,9 @@ public class InMemoryJobRepository implements JobRepository { @Autowired public InMemoryJobRepository(JobManager jobManager) { this.jobManager = jobManager; - this.storage = new HashMap<>(); - // this.storage = - // this.jobManager.listRunningJobs().stream().collect(Collectors.toMap(Job::getId, j -> - // j)); + this.storage = + this.jobManager.listRunningJobs().stream().collect(Collectors.toMap(Job::getId, j -> j)); } /** diff --git a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java new file mode 100644 index 00000000000..1b075835a73 --- /dev/null +++ b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.jc.grpc; + +import feast.jc.dao.JobRepository; +import io.grpc.Status; +import io.grpc.health.v1.HealthGrpc.HealthImplBase; +import io.grpc.health.v1.HealthProto.HealthCheckRequest; +import io.grpc.health.v1.HealthProto.HealthCheckResponse; +import io.grpc.health.v1.HealthProto.HealthCheckResponse.ServingStatus; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.server.service.GrpcService; +import org.springframework.beans.factory.annotation.Autowired; + +@Slf4j +@GrpcService +public class HealthServiceImpl extends HealthImplBase { + private final JobRepository jobRepository; + + @Autowired + public HealthServiceImpl(JobRepository jobRepository) { + this.jobRepository = jobRepository; + } + + @Override + public void check( + HealthCheckRequest request, StreamObserver responseObserver) { + try { + jobRepository.findAll(); + responseObserver.onNext( + HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + log.error("Health Check: unable to retrieve projects.\nError: %s", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } + } +} From abe52b3bd2d0ece0b04ef29082fd15b3b31e8461 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 16:23:07 +0800 Subject: [PATCH 16/26] core-host property --- infra/charts/feast/charts/feast-jc/templates/configmap.yaml | 1 + infra/docker-compose/jc/jc.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml index 196dc5aa989..990702e9d89 100644 --- a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml @@ -13,6 +13,7 @@ data: application-generated.yaml: | {{- if index .Values "application-generated.yaml" "enabled" }} feast: + core-host: {{ .Release.Name }}-feast-core stream: type: kafka options: diff --git a/infra/docker-compose/jc/jc.yml b/infra/docker-compose/jc/jc.yml index cb57ab074f6..4d28d62f8c8 100644 --- a/infra/docker-compose/jc/jc.yml +++ b/infra/docker-compose/jc/jc.yml @@ -1,4 +1,5 @@ feast: + core-host: core jobs: polling_interval_milliseconds: 20000 job_update_timeout_seconds: 240 From 1159f3c30d066673b26245b92492de2eb682c2d2 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 16:53:41 +0800 Subject: [PATCH 17/26] clean --- .../java/feast/core/auth/CoreServiceAuthTest.java | 8 +------- .../feast/core/auth/CoreServiceAuthenticationIT.java | 10 +--------- .../main/java/feast/jc/grpc/HealthServiceImpl.java | 11 ++++++----- .../java/feast/jc/service/JobCoordinatorService.java | 9 +++++---- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java index 262012d0922..949881d5e62 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java @@ -67,7 +67,6 @@ public class CoreServiceAuthTest { @Mock private ProjectRepository projectRepository; @Mock private AuthorizationProvider authProvider; @Mock private StatsService statsService; - // @Mock private JobService jobService; public CoreServiceAuthTest() { MockitoAnnotations.initMocks(this); @@ -83,12 +82,7 @@ public CoreServiceAuthTest() { new AuthorizationService(feastProperties.getSecurity(), authProvider); coreService = new CoreServiceImpl( - specService, - projectService, - statsService, - // jobService, - feastProperties, - authService); + specService, projectService, statsService, feastProperties, authService); } @Test diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index ec8abbbb3db..93b5587051f 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -39,7 +39,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; @@ -52,8 +51,6 @@ public class CoreServiceAuthenticationIT extends BaseIT { @Autowired FeastProperties feastProperties; - @Autowired Map m; - private static int feast_core_port; private static int JWKS_PORT = 45124; @@ -179,12 +176,7 @@ void canApplyFeatureSetIfAuthenticated() { } @TestConfiguration - public static class TestConfig extends BaseTestConfig { - @Bean - public Map testMap() { - return Collections.emptyMap(); - } - } + public static class TestConfig extends BaseTestConfig {} // Create secure Feast Core gRPC client for a specific user private static SimpleCoreClient getSecureApiClient(String subjectEmail) { diff --git a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java index 1b075835a73..5d58cb02d92 100644 --- a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java +++ b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java @@ -16,7 +16,8 @@ */ package feast.jc.grpc; -import feast.jc.dao.JobRepository; +import feast.proto.core.CoreServiceGrpc; +import feast.proto.core.CoreServiceProto; import io.grpc.Status; import io.grpc.health.v1.HealthGrpc.HealthImplBase; import io.grpc.health.v1.HealthProto.HealthCheckRequest; @@ -30,18 +31,18 @@ @Slf4j @GrpcService public class HealthServiceImpl extends HealthImplBase { - private final JobRepository jobRepository; + private final CoreServiceGrpc.CoreServiceBlockingStub specService; @Autowired - public HealthServiceImpl(JobRepository jobRepository) { - this.jobRepository = jobRepository; + public HealthServiceImpl(CoreServiceGrpc.CoreServiceBlockingStub specService) { + this.specService = specService; } @Override public void check( HealthCheckRequest request, StreamObserver responseObserver) { try { - jobRepository.findAll(); + specService.listFeatureSets(CoreServiceProto.ListFeatureSetsRequest.newBuilder().build()); responseObserver.onNext( HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); responseObserver.onCompleted(); diff --git a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java index b50f628389b..fc1216279e5 100644 --- a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java +++ b/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java @@ -321,7 +321,7 @@ private List getAllStores() { .setFilter(Filter.newBuilder().build()) .build()); } catch (StatusRuntimeException e) { - log.error("Core Service is unavailable"); + log.error("Core Service is unavailable. Reason: {}", e.getMessage()); return Collections.emptyList(); } @@ -398,8 +398,8 @@ public void notifyJobsWhenFeatureSetUpdated() { .build()) .build()) .getFeatureSetsList(); - } catch (StatusRuntimeException exc) { - log.error("Core Service is unavailable"); + } catch (StatusRuntimeException e) { + log.error("Core Service is unavailable. Reason: {}", e.getMessage()); return; } @@ -495,7 +495,8 @@ public void listenAckFromJobs( .build()) .getFeatureSet(); } catch (StatusRuntimeException e) { - featureSet = null; + log.error("Core Service is unavailable. Reason: {}", e.getMessage()); + return; } if (featureSet == null) { From e48a86f134e355fc1638bd663d5d072c41676c6f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 19 Aug 2020 17:20:59 +0800 Subject: [PATCH 18/26] more clean up --- .../java/feast/core/model/FeatureSet.java | 20 ------------------- infra/charts/feast/charts/feast-jc/README.md | 10 +++++----- .../charts/feast-jc/templates/configmap.yaml | 2 +- .../charts/feast-jc/templates/deployment.yaml | 6 +++--- .../charts/feast-jc/templates/ingress.yaml | 4 ++-- .../charts/feast-jc/templates/secret.yaml | 2 +- .../charts/feast-jc/templates/service.yaml | 2 +- .../charts/feast/charts/feast-jc/values.yaml | 14 ++++++------- .../java/feast/jc/grpc/HealthServiceImpl.java | 2 +- 9 files changed, 20 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index 4af3217d681..023708d6a99 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -30,7 +30,6 @@ import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.commons.lang3.tuple.Pair; import org.tensorflow.metadata.v0.*; @Getter @@ -231,25 +230,6 @@ public void setProject(Project project) { this.project = project; } - public String getReference() { - return String.format("%s/%s", getProjectName(), getName()); - } - - /** @return Pair <ProjectName, FeatureSetName> */ - public static Pair parseReference(String reference) { - String[] split = reference.split("/", 2); - if (split.length == 1) { - return Pair.of(Project.DEFAULT_NAME, split[0]); - } - - if (split.length > 2) { - throw new RuntimeException( - "FeatureSet reference must have the format /"); - } - - return Pair.of(split[0], split[1]); - } - public int incVersion() { return ++version; } diff --git a/infra/charts/feast/charts/feast-jc/README.md b/infra/charts/feast/charts/feast-jc/README.md index 8860b83bcb9..651f38e2d2b 100644 --- a/infra/charts/feast/charts/feast-jc/README.md +++ b/infra/charts/feast/charts/feast-jc/README.md @@ -13,9 +13,9 @@ Current chart version is `0.7-SNAPSHOT` | Key | Type | Default | Description | |-----|------|---------|-------------| | "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. | -| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | -| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | -| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | +| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | +| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | +| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | | envOverrides | object | `{}` | Extra environment variables to set | | gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow | | gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key | @@ -51,9 +51,9 @@ Current chart version is `0.7-SNAPSHOT` | logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | | logType | string | `"Console"` | Log format, either `JSON` or `Console` | | nodeSelector | object | `{}` | Node labels for pod assignment | -| podLabels | object | `{}` | Labels to be added to Feast Core pods | +| podLabels | object | `{}` | Labels to be added to Feast Job Coordinator pods | | postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres | -| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | +| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Job Coordinator metrics | | readinessProbe.enabled | bool | `true` | Flag to enabled the probe | | readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | | readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated | diff --git a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml index 990702e9d89..7958fd7fff0 100644 --- a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/configmap.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ template "feast-jc.name" . }} - component: core + component: jc chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} diff --git a/infra/charts/feast/charts/feast-jc/templates/deployment.yaml b/infra/charts/feast/charts/feast-jc/templates/deployment.yaml index 151d4ad6c3c..873a9217cb3 100644 --- a/infra/charts/feast/charts/feast-jc/templates/deployment.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ template "feast-jc.name" . }} - component: core + component: jc chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} @@ -14,7 +14,7 @@ spec: selector: matchLabels: app: {{ template "feast-jc.name" . }} - component: core + component: jc release: {{ .Release.Name }} template: metadata: @@ -28,7 +28,7 @@ spec: {{- end }} labels: app: {{ template "feast-jc.name" . }} - component: core + component: jc release: {{ .Release.Name }} {{- if .Values.podLabels }} {{ toYaml .Values.podLabels | nindent 8 }} diff --git a/infra/charts/feast/charts/feast-jc/templates/ingress.yaml b/infra/charts/feast/charts/feast-jc/templates/ingress.yaml index 7f453e1a75f..06ece3078da 100644 --- a/infra/charts/feast/charts/feast-jc/templates/ingress.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/ingress.yaml @@ -1,7 +1,7 @@ {{- if .Values.ingress.http.enabled -}} -{{ template "feast.ingress" (list . "core" "http" .Values.ingress.http) }} +{{ template "feast.ingress" (list . "jc" "http" .Values.ingress.http) }} {{- end }} --- {{ if .Values.ingress.grpc.enabled -}} -{{ template "feast.ingress" (list . "core" "grpc" .Values.ingress.grpc) }} +{{ template "feast.ingress" (list . "jc" "grpc" .Values.ingress.grpc) }} {{- end }} diff --git a/infra/charts/feast/charts/feast-jc/templates/secret.yaml b/infra/charts/feast/charts/feast-jc/templates/secret.yaml index d10f8614558..b48c7b2833b 100644 --- a/infra/charts/feast/charts/feast-jc/templates/secret.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/secret.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ template "feast-jc.name" . }} - component: core + component: jc chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} diff --git a/infra/charts/feast/charts/feast-jc/templates/service.yaml b/infra/charts/feast/charts/feast-jc/templates/service.yaml index 2b4c49027f4..de3801a6b2c 100644 --- a/infra/charts/feast/charts/feast-jc/templates/service.yaml +++ b/infra/charts/feast/charts/feast-jc/templates/service.yaml @@ -36,6 +36,6 @@ spec: {{- end }} selector: app: {{ template "feast-jc.name" . }} - component: core + component: jc release: {{ .Release.Name }} diff --git a/infra/charts/feast/charts/feast-jc/values.yaml b/infra/charts/feast/charts/feast-jc/values.yaml index 18ee7de9cf1..41e679494b0 100644 --- a/infra/charts/feast/charts/feast-jc/values.yaml +++ b/infra/charts/feast/charts/feast-jc/values.yaml @@ -10,18 +10,18 @@ image: pullPolicy: IfNotPresent application.yaml: - # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. + # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. enabled: true application-generated.yaml: # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. enabled: true -# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. +# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. application-secret.yaml: enabled: true -# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` +# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` application-override.yaml: enabled: true @@ -50,14 +50,12 @@ logType: Console logLevel: WARN prometheus: - # prometheus.enabled -- Flag to enable scraping of Feast Core metrics + # prometheus.enabled -- Flag to enable scraping of metrics enabled: true -# By default we disable the liveness probe, since if the DB fails restarting core will not result -# in application healing. livenessProbe: # livenessProbe.enabled -- Flag to enabled the probe - enabled: false + enabled: true # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated initialDelaySeconds: 60 # livenessProbe.periodSeconds -- How often to perform the probe @@ -152,5 +150,5 @@ nodeSelector: {} # envOverrides -- Extra environment variables to set envOverrides: {} -# podLabels -- Labels to be added to Feast Core pods +# podLabels -- Labels to be added to Feast Job Coordinator pods podLabels: {} diff --git a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java index 5d58cb02d92..3b1f067e668 100644 --- a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java +++ b/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java @@ -42,7 +42,7 @@ public HealthServiceImpl(CoreServiceGrpc.CoreServiceBlockingStub specService) { public void check( HealthCheckRequest request, StreamObserver responseObserver) { try { - specService.listFeatureSets(CoreServiceProto.ListFeatureSetsRequest.newBuilder().build()); + specService.listProjects(CoreServiceProto.ListProjectsRequest.newBuilder().build()); responseObserver.onNext( HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); responseObserver.onCompleted(); From b90a0b624ce1175dc9b533b5d8aaf10887230bad Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 20 Aug 2020 14:17:35 +0800 Subject: [PATCH 19/26] format --- common-test/pom.xml | 12 ++++++------ infra/scripts/validate-version-consistency.sh | 3 +++ protos/feast/core/CoreService.proto | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 3a2d44d5eb0..563129cf063 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -44,19 +44,19 @@ - + dev.feast datatypes-java ${project.version} compile - + dev.feast feast-common ${project.version} compile - + com.google.protobuf protobuf-java-util @@ -67,9 +67,9 @@ ${lombok.version} - javax.validation - validation-api - + javax.validation + validation-api + com.google.auto.value auto-value-annotations diff --git a/infra/scripts/validate-version-consistency.sh b/infra/scripts/validate-version-consistency.sh index 6f9786b207e..a3138ed8f62 100755 --- a/infra/scripts/validate-version-consistency.sh +++ b/infra/scripts/validate-version-consistency.sh @@ -41,6 +41,9 @@ declare -a files_to_validate_version=( "infra/charts/feast/charts/feast-jupyter/Chart.yaml,1,${FEAST_MASTER_VERSION}" "infra/charts/feast/charts/feast-serving/values.yaml,1,${FEAST_RELEASE_VERSION}" "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_RELEASE_VERSION}" + "infra/charts/feast/charts/feast-jc/Chart.yaml,1,${FEAST_MASTER_VERSION}" + "infra/charts/feast/charts/feast-jc/values.yaml,1,${FEAST_RELEASE_VERSION}" + "infra/charts/feast/charts/feast-jc/README.md,1,${FEAST_RELEASE_VERSION}" "infra/charts/feast/requirements.yaml,4,${FEAST_MASTER_VERSION}" "infra/charts/feast/requirements.lock,4,${FEAST_RELEASE_VERSION}" "infra/docker-compose/.env.sample,1,${FEAST_RELEASE_VERSION}" diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index bdb35d97b38..658b4940b52 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -92,7 +92,7 @@ service CoreService { rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse); // Internal API for Job Coordinator to update featureSet's status once responsible ingestion job is running - rpc UpdateFeatureSetStatus(UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); + rpc UpdateFeatureSetStatus (UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); } @@ -101,18 +101,18 @@ service JobCoordinatorService { // Returns allow ingestions matching the given request filter. // Returns all ingestion jobs if no filter is provided. // Returns an empty list if no ingestion jobs match the filter. - rpc ListIngestionJobs(ListIngestionJobsRequest) returns (ListIngestionJobsResponse); + rpc ListIngestionJobs (ListIngestionJobsRequest) returns (ListIngestionJobsResponse); // Restart an Ingestion Job. Restarts the ingestion job with the given job id. // NOTE: Data might be lost during the restart for some job runners. // Does not support stopping a job in a transitional (ie pending, suspending, aborting), // terminal state (ie suspended or aborted) or unknown status - rpc RestartIngestionJob(RestartIngestionJobRequest) returns (RestartIngestionJobResponse); + rpc RestartIngestionJob (RestartIngestionJobRequest) returns (RestartIngestionJobResponse); // Stop an Ingestion Job. Stop (Aborts) the ingestion job with the given job id. // Does nothing if the target job if already in a terminal state (ie suspended or aborted). // Does not support stopping a job in a transitional (ie pending, suspending, aborting) or unknown status - rpc StopIngestionJob(StopIngestionJobRequest) returns (StopIngestionJobResponse); + rpc StopIngestionJob (StopIngestionJobRequest) returns (StopIngestionJobResponse); } // Request for a single feature set From 3bf9f1a4e7be7375dda80ac1d94e612759f175b0 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 20 Aug 2020 15:49:02 +0800 Subject: [PATCH 20/26] rename jc -> jobcontroller --- .github/workflows/complete.yml | 2 +- .github/workflows/master_only.yml | 2 +- Makefile | 12 +-- .../java/feast/common/it/SimpleJCClient.java | 6 +- infra/charts/feast/charts/feast-jc/Chart.yaml | 4 - .../charts/feast-jc/templates/ingress.yaml | 7 -- .../charts/feast-jobcontroller/Chart.yaml | 4 + .../README.md | 16 ++-- .../templates/_helpers.tpl | 12 +-- .../templates/_ingress.yaml | 0 .../templates/configmap.yaml | 6 +- .../templates/deployment.yaml | 32 ++++---- .../templates/ingress.yaml | 7 ++ .../templates/secret.yaml | 6 +- .../templates/service.yaml | 8 +- .../values.yaml | 10 +-- .../charts/feast/values-dataflow-runner.yaml | 2 +- infra/charts/feast/values.yaml | 4 +- infra/docker-compose/.env.sample | 2 +- infra/docker-compose/docker-compose.yml | 8 +- .../{jc => jobcontroller}/jc.yml | 0 infra/docker/jc/Dockerfile.dev | 8 -- infra/docker/{jc => jobcontroller}/Dockerfile | 14 ++-- .../{jc => jobcontroller}/Dockerfile.debug | 0 infra/docker/jobcontroller/Dockerfile.dev | 8 ++ infra/scripts/setup-common-functions.sh | 10 +-- .../scripts/test-end-to-end-batch-dataflow.sh | 4 +- infra/scripts/test-end-to-end-batch.sh | 2 +- .../scripts/test-end-to-end-redis-cluster.sh | 2 +- infra/scripts/test-end-to-end.sh | 2 +- infra/scripts/test-load.sh | 10 +-- .../values-end-to-end-batch-dataflow.yaml | 4 +- infra/scripts/validate-version-consistency.sh | 6 +- {job-coordinator => job-controller}/pom.xml | 6 +- .../JobControllerApplication.java | 8 +- .../config/FeastProperties.java | 18 ++--- .../config/FeatureStreamConfig.java | 2 +- .../config/JobControllerConfig.java | 24 +++--- .../dao/InMemoryJobRepository.java | 8 +- .../jobcontroller}/dao/JobRepository.java | 6 +- .../exception/JobExecutionException.java | 2 +- .../exception/JobMonitoringException.java | 2 +- .../jobcontroller}/grpc/CoreServiceImpl.java | 8 +- .../grpc/HealthServiceImpl.java | 2 +- .../model/FeatureSetDeliveryStatus.java | 2 +- .../java/feast/jobcontroller}/model/Job.java | 2 +- .../feast/jobcontroller}/model/JobStatus.java | 2 +- .../runner/ConsolidatedJobStrategy.java | 8 +- .../runner/JobGroupingStrategy.java | 4 +- .../jobcontroller}/runner/JobManager.java | 6 +- .../runner/JobPerStoreStrategy.java | 8 +- .../feast/jobcontroller}/runner/Runner.java | 2 +- .../runner/dataflow/DataflowJobManager.java | 16 ++-- .../runner/dataflow/DataflowJobState.java | 2 +- .../dataflow/DataflowJobStateMapper.java | 6 +- .../runner/dataflow/DataflowJobType.java | 2 +- .../runner/dataflow/DataflowRunnerConfig.java | 4 +- .../runner/direct/DirectJob.java | 2 +- .../runner/direct/DirectJobRegistry.java | 2 +- .../runner/direct/DirectJobStateMapper.java | 4 +- .../runner/direct/DirectRunnerConfig.java | 4 +- .../runner/direct/DirectRunnerJobManager.java | 14 ++-- .../option/FeatureSetJsonByteConverter.java | 2 +- .../runner/option/RunnerConfig.java | 6 +- .../runner/task/CreateJobTask.java | 8 +- .../jobcontroller}/runner/task/JobTask.java | 8 +- .../jobcontroller}/runner/task/JobTasks.java | 2 +- .../runner/task/RestartJobTask.java | 8 +- .../runner/task/TerminateJobTask.java | 8 +- .../runner/task/UpdateJobStatusTask.java | 8 +- .../service/JobControllerService.java | 38 ++++----- .../jobcontroller}/service/JobService.java | 14 ++-- .../jobcontroller}/util/PackageUtil.java | 2 +- .../jobcontroller}/util/PipelineUtil.java | 4 +- .../jobcontroller}/util/TypeConversion.java | 2 +- .../src/main/resources/application.yml | 2 +- .../src/main/resources/banner.txt | 0 .../jobcontroller}/model/JobStatusTest.java | 2 +- .../jobcontroller}/runner/RunnerTest.java | 2 +- .../dataflow/DataflowJobManagerTest.java | 8 +- .../dataflow/DataflowJobStateMapperTest.java | 2 +- .../dataflow/DataflowRunnerConfigTest.java | 2 +- .../runner/direct/DirectRunnerConfigTest.java | 2 +- .../direct/DirectRunnerJobManagerTest.java | 8 +- .../FeatureSetJsonByteConverterTest.java | 2 +- .../runner/task/JobTasksTest.java | 10 +-- .../service/FakeJobManager.java | 10 +-- .../service/JobControllerIT.java | 32 ++++---- .../service/JobControllerServiceTest.java | 78 +++++++++---------- .../jobcontroller}/service/JobServiceIT.java | 18 ++--- .../test/resources/application-it-core.yml | 0 .../test/resources/application-it.properties | 0 .../org.mockito.plugins.MockMaker | 0 pom.xml | 2 +- protos/feast/core/CoreService.proto | 4 +- sdk/python/feast/cli.py | 10 +-- sdk/python/feast/constants.py | 6 +- .../__init__.py | 0 .../client.py | 32 ++++---- .../job.py | 4 +- sdk/python/tests/test_client.py | 50 ++++++------ tests/e2e/bq/bq-batch-retrieval.py | 10 +-- tests/e2e/conftest.py | 2 +- tests/e2e/redis/basic-ingest-redis-serving.py | 30 +++---- 104 files changed, 417 insertions(+), 417 deletions(-) delete mode 100644 infra/charts/feast/charts/feast-jc/Chart.yaml delete mode 100644 infra/charts/feast/charts/feast-jc/templates/ingress.yaml create mode 100644 infra/charts/feast/charts/feast-jobcontroller/Chart.yaml rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/README.md (86%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/_helpers.tpl (79%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/_ingress.yaml (100%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/configmap.yaml (87%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/deployment.yaml (83%) create mode 100644 infra/charts/feast/charts/feast-jobcontroller/templates/ingress.yaml rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/secret.yaml (71%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/templates/service.yaml (85%) rename infra/charts/feast/charts/{feast-jc => feast-jobcontroller}/values.yaml (89%) rename infra/docker-compose/{jc => jobcontroller}/jc.yml (100%) delete mode 100644 infra/docker/jc/Dockerfile.dev rename infra/docker/{jc => jobcontroller}/Dockerfile (82%) rename infra/docker/{jc => jobcontroller}/Dockerfile.debug (100%) create mode 100644 infra/docker/jobcontroller/Dockerfile.dev rename {job-coordinator => job-controller}/pom.xml (98%) rename job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java => job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java (89%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/config/FeastProperties.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/config/FeatureStreamConfig.java (99%) rename job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java => job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java (89%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/dao/InMemoryJobRepository.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/dao/JobRepository.java (92%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/exception/JobExecutionException.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/exception/JobMonitoringException.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/grpc/CoreServiceImpl.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/grpc/HealthServiceImpl.java (98%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/model/FeatureSetDeliveryStatus.java (98%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/model/Job.java (99%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/model/JobStatus.java (99%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/ConsolidatedJobStrategy.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/JobGroupingStrategy.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/JobManager.java (94%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/JobPerStoreStrategy.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/Runner.java (97%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/dataflow/DataflowJobManager.java (97%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/dataflow/DataflowJobState.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/dataflow/DataflowJobStateMapper.java (93%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/dataflow/DataflowJobType.java (94%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/dataflow/DataflowRunnerConfig.java (97%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/direct/DirectJob.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/direct/DirectJobRegistry.java (97%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/direct/DirectJobStateMapper.java (94%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/direct/DirectRunnerConfig.java (93%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/direct/DirectRunnerJobManager.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/option/FeatureSetJsonByteConverter.java (97%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/option/RunnerConfig.java (92%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/CreateJobTask.java (92%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/JobTask.java (92%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/JobTasks.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/RestartJobTask.java (90%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/TerminateJobTask.java (89%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/runner/task/UpdateJobStatusTask.java (87%) rename job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java => job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java (95%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/service/JobService.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/util/PackageUtil.java (99%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/util/PipelineUtil.java (96%) rename {job-coordinator/src/main/java/feast/jc => job-controller/src/main/java/feast/jobcontroller}/util/TypeConversion.java (98%) rename {job-coordinator => job-controller}/src/main/resources/application.yml (99%) rename {job-coordinator => job-controller}/src/main/resources/banner.txt (100%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/model/JobStatusTest.java (97%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/RunnerTest.java (97%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/dataflow/DataflowJobManagerTest.java (98%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/dataflow/DataflowJobStateMapperTest.java (95%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/dataflow/DataflowRunnerConfigTest.java (99%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/direct/DirectRunnerConfigTest.java (97%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/direct/DirectRunnerJobManagerTest.java (97%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/option/FeatureSetJsonByteConverterTest.java (98%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/runner/task/JobTasksTest.java (95%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/service/FakeJobManager.java (89%) rename job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java => job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java (93%) rename job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java => job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java (85%) rename {job-coordinator/src/test/java/feast/jc => job-controller/src/test/java/feast/jobcontroller}/service/JobServiceIT.java (94%) rename {job-coordinator => job-controller}/src/test/resources/application-it-core.yml (100%) rename {job-coordinator => job-controller}/src/test/resources/application-it.properties (100%) rename {job-coordinator => job-controller}/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker (100%) rename sdk/python/feast/contrib/{job_coordinator => job_controller}/__init__.py (100%) rename sdk/python/feast/contrib/{job_coordinator => job_controller}/client.py (80%) rename sdk/python/feast/contrib/{job_coordinator => job_controller}/job.py (96%) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 7565ba70f16..cb3467b1d03 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -7,7 +7,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jc, jupyter] + component: [core, serving, jobcontroller, jupyter] env: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} REGISTRY: gcr.io/kf-feast diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index e144e23fb78..eae6119e343 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jc, jupyter, ci] + component: [core, serving, jobcontroller, jupyter, ci] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar steps: diff --git a/Makefile b/Makefile index 8af2343c59f..be0482de6e7 100644 --- a/Makefile +++ b/Makefile @@ -117,15 +117,15 @@ build-push-docker: @$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION) @$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION) @$(MAKE) push-ci-docker registry=$(REGISTRY) version=$(VERSION) - @$(MAKE) push-jc-docker registry=$(REGISTRY) version=$(VERSION) + @$(MAKE) push-jobcontroller-docker registry=$(REGISTRY) version=$(VERSION) -build-docker: build-core-docker build-serving-docker build-ci-docker build-jc-docker +build-docker: build-core-docker build-serving-docker build-ci-docker build-jobcontroller-docker push-core-docker: docker push $(REGISTRY)/feast-core:$(VERSION) -push-jc-docker: - docker push $(REGISTRY)/feast-jc:$(VERSION) +push-jobcontroller-docker: + docker push $(REGISTRY)/feast-jobcontroller:$(VERSION) push-serving-docker: docker push $(REGISTRY)/feast-serving:$(VERSION) @@ -139,8 +139,8 @@ push-jupyter-docker: build-core-docker: docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . -build-jc-docker: - docker build -t $(REGISTRY)/feast-jc:$(VERSION) -f infra/docker/jc/Dockerfile . +build-jobcontroller-docker: + docker build -t $(REGISTRY)/feast-jobcontroller:$(VERSION) -f infra/docker/jobcontroller/Dockerfile . build-serving-docker: docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . diff --git a/common-test/src/main/java/feast/common/it/SimpleJCClient.java b/common-test/src/main/java/feast/common/it/SimpleJCClient.java index 99bfb349c56..31c62934f7f 100644 --- a/common-test/src/main/java/feast/common/it/SimpleJCClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleJCClient.java @@ -18,13 +18,13 @@ import feast.proto.core.CoreServiceProto; import feast.proto.core.IngestionJobProto; -import feast.proto.core.JobCoordinatorServiceGrpc; +import feast.proto.core.JobControllerServiceGrpc; import java.util.List; public class SimpleJCClient { - private final JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub; + private final JobControllerServiceGrpc.JobControllerServiceBlockingStub stub; - public SimpleJCClient(JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub) { + public SimpleJCClient(JobControllerServiceGrpc.JobControllerServiceBlockingStub stub) { this.stub = stub; } diff --git a/infra/charts/feast/charts/feast-jc/Chart.yaml b/infra/charts/feast/charts/feast-jc/Chart.yaml deleted file mode 100644 index b10c7a811b9..00000000000 --- a/infra/charts/feast/charts/feast-jc/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Feast Job Coordinator manage ingestion jobs. -name: feast-jc -version: 0.7-SNAPSHOT diff --git a/infra/charts/feast/charts/feast-jc/templates/ingress.yaml b/infra/charts/feast/charts/feast-jc/templates/ingress.yaml deleted file mode 100644 index 06ece3078da..00000000000 --- a/infra/charts/feast/charts/feast-jc/templates/ingress.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if .Values.ingress.http.enabled -}} -{{ template "feast.ingress" (list . "jc" "http" .Values.ingress.http) }} -{{- end }} ---- -{{ if .Values.ingress.grpc.enabled -}} -{{ template "feast.ingress" (list . "jc" "grpc" .Values.ingress.grpc) }} -{{- end }} diff --git a/infra/charts/feast/charts/feast-jobcontroller/Chart.yaml b/infra/charts/feast/charts/feast-jobcontroller/Chart.yaml new file mode 100644 index 00000000000..ae0aad538ff --- /dev/null +++ b/infra/charts/feast/charts/feast-jobcontroller/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Feast Job Coontroller manage ingestion jobs. +name: feast-jobcontroller +version: 0.7-SNAPSHOT diff --git a/infra/charts/feast/charts/feast-jc/README.md b/infra/charts/feast/charts/feast-jobcontroller/README.md similarity index 86% rename from infra/charts/feast/charts/feast-jc/README.md rename to infra/charts/feast/charts/feast-jobcontroller/README.md index 651f38e2d2b..7c27fccedf6 100644 --- a/infra/charts/feast/charts/feast-jc/README.md +++ b/infra/charts/feast/charts/feast-jobcontroller/README.md @@ -1,6 +1,6 @@ -feast-jc +feast-jobcontroller ========== -Feast Job Coordinator manage ingestion jobs. +Feast Job Controller manage ingestion jobs. Current chart version is `0.7-SNAPSHOT` @@ -13,16 +13,16 @@ Current chart version is `0.7-SNAPSHOT` | Key | Type | Default | Description | |-----|------|---------|-------------| | "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. | -| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | -| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | -| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | +| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | +| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | +| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | | envOverrides | object | `{}` | Extra environment variables to set | | gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow | | gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key | | gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-jc"` | Docker image repository | +| image.repository | string | `"gcr.io/kf-feast/feast-jobcontroller"` | Docker image repository | | image.tag | string | `"0.6.2"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | @@ -51,9 +51,9 @@ Current chart version is `0.7-SNAPSHOT` | logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | | logType | string | `"Console"` | Log format, either `JSON` or `Console` | | nodeSelector | object | `{}` | Node labels for pod assignment | -| podLabels | object | `{}` | Labels to be added to Feast Job Coordinator pods | +| podLabels | object | `{}` | Labels to be added to Feast Job Controller pods | | postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres | -| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Job Coordinator metrics | +| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Job Controller metrics | | readinessProbe.enabled | bool | `true` | Flag to enabled the probe | | readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | | readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated | diff --git a/infra/charts/feast/charts/feast-jc/templates/_helpers.tpl b/infra/charts/feast/charts/feast-jobcontroller/templates/_helpers.tpl similarity index 79% rename from infra/charts/feast/charts/feast-jc/templates/_helpers.tpl rename to infra/charts/feast/charts/feast-jobcontroller/templates/_helpers.tpl index 8e18ba61d62..3fdbaab59a3 100644 --- a/infra/charts/feast/charts/feast-jc/templates/_helpers.tpl +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "feast-jc.name" -}} +{{- define "feast-jobcontroller.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -11,7 +11,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "feast-jc.fullname" -}} +{{- define "feast-jobcontroller.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -27,16 +27,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "feast-jc.chart" -}} +{{- define "feast-jobcontroller.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Common labels */}} -{{- define "feast-jc.labels" -}} -app.kubernetes.io/name: {{ include "feast-jc.name" . }} -helm.sh/chart: {{ include "feast-jc.chart" . }} +{{- define "feast-jobcontroller.labels" -}} +app.kubernetes.io/name: {{ include "feast-jobcontroller.name" . }} +helm.sh/chart: {{ include "feast-jobcontroller.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} diff --git a/infra/charts/feast/charts/feast-jc/templates/_ingress.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/_ingress.yaml similarity index 100% rename from infra/charts/feast/charts/feast-jc/templates/_ingress.yaml rename to infra/charts/feast/charts/feast-jobcontroller/templates/_ingress.yaml diff --git a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/configmap.yaml similarity index 87% rename from infra/charts/feast/charts/feast-jc/templates/configmap.yaml rename to infra/charts/feast/charts/feast-jobcontroller/templates/configmap.yaml index 7958fd7fff0..b92c6220d63 100644 --- a/infra/charts/feast/charts/feast-jc/templates/configmap.yaml +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/configmap.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "feast-jc.fullname" . }} + name: {{ template "feast-jobcontroller.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} diff --git a/infra/charts/feast/charts/feast-jc/templates/deployment.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/deployment.yaml similarity index 83% rename from infra/charts/feast/charts/feast-jc/templates/deployment.yaml rename to infra/charts/feast/charts/feast-jobcontroller/templates/deployment.yaml index 873a9217cb3..c9a33dbe281 100644 --- a/infra/charts/feast/charts/feast-jc/templates/deployment.yaml +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/deployment.yaml @@ -1,11 +1,11 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ template "feast-jc.fullname" . }} + name: {{ template "feast-jobcontroller.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} @@ -13,8 +13,8 @@ spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller release: {{ .Release.Name }} template: metadata: @@ -27,8 +27,8 @@ spec: prometheus.io/scrape: "true" {{- end }} labels: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller release: {{ .Release.Name }} {{- if .Values.podLabels }} {{ toYaml .Values.podLabels | nindent 8 }} @@ -40,14 +40,14 @@ spec: {{- end }} volumes: - - name: {{ template "feast-jc.fullname" . }}-config + - name: {{ template "feast-jobcontroller.fullname" . }}-config configMap: - name: {{ template "feast-jc.fullname" . }} - - name: {{ template "feast-jc.fullname" . }}-secret + name: {{ template "feast-jobcontroller.fullname" . }} + - name: {{ template "feast-jobcontroller.fullname" . }}-secret secret: - secretName: {{ template "feast-jc.fullname" . }} + secretName: {{ template "feast-jobcontroller.fullname" . }} {{- if .Values.gcpServiceAccount.enabled }} - - name: {{ template "feast-jc.fullname" . }}-gcp-service-account + - name: {{ template "feast-jobcontroller.fullname" . }}-gcp-service-account secret: secretName: {{ .Values.gcpServiceAccount.existingSecret.name }} {{- end }} @@ -58,13 +58,13 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - - name: {{ template "feast-jc.fullname" . }}-config + - name: {{ template "feast-jobcontroller.fullname" . }}-config mountPath: /etc/feast - - name: {{ template "feast-jc.fullname" . }}-secret + - name: {{ template "feast-jobcontroller.fullname" . }}-secret mountPath: /etc/secrets/feast readOnly: true {{- if .Values.gcpServiceAccount.enabled }} - - name: {{ template "feast-jc.fullname" . }}-gcp-service-account + - name: {{ template "feast-jobcontroller.fullname" . }}-gcp-service-account mountPath: /etc/secrets/google readOnly: true {{- end }} @@ -106,7 +106,7 @@ spec: command: - java - -jar - - /opt/feast/feast-job-coordinator.jar + - /opt/feast/feast-job-controller.jar - --spring.config.location= {{- if index .Values "application.yaml" "enabled" -}} classpath:/application.yml diff --git a/infra/charts/feast/charts/feast-jobcontroller/templates/ingress.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/ingress.yaml new file mode 100644 index 00000000000..b30decd3063 --- /dev/null +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/ingress.yaml @@ -0,0 +1,7 @@ +{{- if .Values.ingress.http.enabled -}} +{{ template "feast.ingress" (list . "jobcontroller" "http" .Values.ingress.http) }} +{{- end }} +--- +{{ if .Values.ingress.grpc.enabled -}} +{{ template "feast.ingress" (list . "jobcontroller" "grpc" .Values.ingress.grpc) }} +{{- end }} diff --git a/infra/charts/feast/charts/feast-jc/templates/secret.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/secret.yaml similarity index 71% rename from infra/charts/feast/charts/feast-jc/templates/secret.yaml rename to infra/charts/feast/charts/feast-jobcontroller/templates/secret.yaml index b48c7b2833b..b3cab27178e 100644 --- a/infra/charts/feast/charts/feast-jc/templates/secret.yaml +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/secret.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Secret metadata: - name: {{ template "feast-jc.fullname" . }} + name: {{ template "feast-jobcontroller.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} diff --git a/infra/charts/feast/charts/feast-jc/templates/service.yaml b/infra/charts/feast/charts/feast-jobcontroller/templates/service.yaml similarity index 85% rename from infra/charts/feast/charts/feast-jc/templates/service.yaml rename to infra/charts/feast/charts/feast-jobcontroller/templates/service.yaml index de3801a6b2c..1d5fbdd7a75 100644 --- a/infra/charts/feast/charts/feast-jc/templates/service.yaml +++ b/infra/charts/feast/charts/feast-jobcontroller/templates/service.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "feast-jc.fullname" . }} + name: {{ template "feast-jobcontroller.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: {{ template "feast-jc.name" . }} + app: {{ template "feast-jobcontroller.name" . }} chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} @@ -35,7 +35,7 @@ spec: nodePort: {{ .Values.service.grpc.nodePort }} {{- end }} selector: - app: {{ template "feast-jc.name" . }} - component: jc + app: {{ template "feast-jobcontroller.name" . }} + component: jobcontroller release: {{ .Release.Name }} diff --git a/infra/charts/feast/charts/feast-jc/values.yaml b/infra/charts/feast/charts/feast-jobcontroller/values.yaml similarity index 89% rename from infra/charts/feast/charts/feast-jc/values.yaml rename to infra/charts/feast/charts/feast-jobcontroller/values.yaml index 41e679494b0..bd4856ad54a 100644 --- a/infra/charts/feast/charts/feast-jc/values.yaml +++ b/infra/charts/feast/charts/feast-jobcontroller/values.yaml @@ -3,25 +3,25 @@ replicaCount: 1 image: # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-jc + repository: gcr.io/kf-feast/feast-jobcontroller # image.tag -- Image tag tag: 0.6.2 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent application.yaml: - # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. + # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. enabled: true application-generated.yaml: # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. enabled: true -# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. +# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. application-secret.yaml: enabled: true -# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-coordinator/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` +# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/job-controller/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` application-override.yaml: enabled: true @@ -150,5 +150,5 @@ nodeSelector: {} # envOverrides -- Extra environment variables to set envOverrides: {} -# podLabels -- Labels to be added to Feast Job Coordinator pods +# podLabels -- Labels to be added to Feast Job Controller pods podLabels: {} diff --git a/infra/charts/feast/values-dataflow-runner.yaml b/infra/charts/feast/values-dataflow-runner.yaml index d5a290e362c..1e95c2bb8ad 100644 --- a/infra/charts/feast/values-dataflow-runner.yaml +++ b/infra/charts/feast/values-dataflow-runner.yaml @@ -10,7 +10,7 @@ feast-core: options: bootstrapServers: -feast-jc: +feast-jobcontroller: gcpServiceAccount: enabled: true application-override.yaml: diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index 94a4b91cb28..b3a2df67bef 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -2,8 +2,8 @@ feast-core: # feast-core.enabled -- Flag to install Feast Core enabled: true -feast-jc: - # feast-jc.enabled -- Flag to install Feast Job Coordinator +feast-jobcontroller: + # feast-jobcontroller.enabled -- Flag to install Feast Job Controller enabled: true feast-online-serving: diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index feedd4a3262..f2d02ceadee 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,7 +2,7 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=0.6.2 GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json FEAST_CORE_CONFIG=./core/core.yml -FEAST_JC_CONFIG=./jc/jc.yml +FEAST_jobcontroller_CONFIG=./jobcontroller/jobcontroller.yml FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml FEAST_HISTORICAL_SERVING_ENABLED=false FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 4d5225b7e43..0efb9eb3296 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -21,10 +21,10 @@ services: - /opt/feast/feast-core.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - jc: - image: gcr.io/kf-feast/feast-jc:${FEAST_VERSION} + jobcontroller: + image: gcr.io/kf-feast/feast-jobcontroller:${FEAST_VERSION} volumes: - - ${FEAST_JC_CONFIG}:/etc/feast/application.yml + - ${FEAST_jobcontroller_CONFIG}:/etc/feast/application.yml - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json environment: GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json @@ -35,7 +35,7 @@ services: command: - java - -jar - - /opt/feast/feast-job-coordinator.jar + - /opt/feast/feast-job-controller.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml jupyter: diff --git a/infra/docker-compose/jc/jc.yml b/infra/docker-compose/jobcontroller/jc.yml similarity index 100% rename from infra/docker-compose/jc/jc.yml rename to infra/docker-compose/jobcontroller/jc.yml diff --git a/infra/docker/jc/Dockerfile.dev b/infra/docker/jc/Dockerfile.dev deleted file mode 100644 index 294926fa391..00000000000 --- a/infra/docker/jc/Dockerfile.dev +++ /dev/null @@ -1,8 +0,0 @@ -FROM openjdk:11-jre -ARG REVISION=dev -ADD $PWD/core/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-job-coordinator.jar"] diff --git a/infra/docker/jc/Dockerfile b/infra/docker/jobcontroller/Dockerfile similarity index 82% rename from infra/docker/jc/Dockerfile rename to infra/docker/jobcontroller/Dockerfile index 2b10b9d3e42..95ad415fbaf 100644 --- a/infra/docker/jc/Dockerfile +++ b/infra/docker/jobcontroller/Dockerfile @@ -18,7 +18,7 @@ COPY storage/connectors/pom.xml storage/connectors/pom.xml COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml COPY storage/connectors/bigquery/pom.xml storage/connectors/bigquery/pom.xml COPY sdk/java/pom.xml sdk/java/pom.xml -COPY job-coordinator/pom.xml job-coordinator/pom.xml +COPY job-controller/pom.xml job-controller/pom.xml COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml COPY protos/ protos/ @@ -32,18 +32,18 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . ARG REVISION=dev -RUN mvn --also-make --projects job-coordinator,ingestion -Drevision=$REVISION \ +RUN mvn --also-make --projects job-controller,ingestion -Drevision=$REVISION \ -DskipUTs=true --batch-mode clean package # # Unpack the jar and copy the files into production Docker image -# for faster startup time when starting Dataflow jobs from Feast JC. +# for faster startup time when starting Dataflow jobs from Feast Job Controller. # This is because we need to stage the classes and dependencies when using Dataflow. # The final size of the production image will be bigger but it seems # a good tradeoff between speed and size. # # https://github.com/feast-dev/feast/pull/291 RUN apt-get -qq update && apt-get -y install unar && \ - unar /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar -o /build/job-coordinator/target/ + unar /build/job-controller/target/feast-job-controller-$REVISION-exec.jar -o /build/job-controller/target/ # # Download grpc_health_probe to run health check for Feast Serving @@ -60,13 +60,13 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa FROM openjdk:11-jre as production ARG REVISION=dev -COPY --from=builder /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec.jar /opt/feast/feast-job-coordinator.jar +COPY --from=builder /build/job-controller/target/feast-job-controller-$REVISION-exec.jar /opt/feast/feast-job-controller.jar # Required for staging jar dependencies when submitting Dataflow jobs. -COPY --from=builder /build/job-coordinator/target/feast-job-coordinator-$REVISION-exec /opt/feast/feast-job-coordinator +COPY --from=builder /build/job-controller/target/feast-job-controller-$REVISION-exec /opt/feast/feast-job-controller COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ "-Xms2048m",\ "-Xmx2048m",\ "-jar",\ - "/opt/feast/feast-job-coordinator.jar"] + "/opt/feast/feast-job-controller.jar"] diff --git a/infra/docker/jc/Dockerfile.debug b/infra/docker/jobcontroller/Dockerfile.debug similarity index 100% rename from infra/docker/jc/Dockerfile.debug rename to infra/docker/jobcontroller/Dockerfile.debug diff --git a/infra/docker/jobcontroller/Dockerfile.dev b/infra/docker/jobcontroller/Dockerfile.dev new file mode 100644 index 00000000000..da238dd2235 --- /dev/null +++ b/infra/docker/jobcontroller/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM openjdk:11-jre +ARG REVISION=dev +ADD $PWD/core/target/feast-job-controller-$REVISION-exec.jar /opt/feast/feast-job-controller.jar +CMD ["java",\ + "-Xms2048m",\ + "-Xmx2048m",\ + "-jar",\ + "/opt/feast/feast-job-controller.jar"] diff --git a/infra/scripts/setup-common-functions.sh b/infra/scripts/setup-common-functions.sh index 445a3c8cc51..ca65e039f0e 100755 --- a/infra/scripts/setup-common-functions.sh +++ b/infra/scripts/setup-common-functions.sh @@ -76,7 +76,7 @@ build_feast_core_and_serving() { ls -lh core/target/*jar ls -lh serving/target/*jar - ls -lh job-coordinator/target/*jar + ls -lh job-controller/target/*jar } start_feast_core() { @@ -94,18 +94,18 @@ start_feast_core() { nc -w2 localhost 6565 /var/log/feast-jc.log & + nohup java -jar job-controller/target/feast-job-controller-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-jobcontroller.log & ${SCRIPTS_DIR}/wait-for-it.sh localhost:6570 --timeout=90 - tail -n10 /var/log/feast-jc.log + tail -n10 /var/log/feast-jobcontroller.log nc -w2 localhost 6570 /tmp/serving.online.application.yml feast: diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 19f683a85b8..8f05efa9df2 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -105,7 +105,7 @@ if [[ ${ENABLE_AUTH} = "true" ]]; print_banner "Starting Feast Serving without auth" fi -start_feast_jc /tmp/jc.warehouse.application.yml +start_feast_jobcontroller /tmp/jc.warehouse.application.yml start_feast_serving /tmp/serving.warehouse.application.yml install_python_with_miniconda_and_feast_sdk diff --git a/infra/scripts/test-load.sh b/infra/scripts/test-load.sh index 88842654173..e8dbebb2c24 100755 --- a/infra/scripts/test-load.sh +++ b/infra/scripts/test-load.sh @@ -36,7 +36,7 @@ else fi wait_for_docker_image gcr.io/kf-feast/feast-core:"${FEAST_VERSION}" -wait_for_docker_image gcr.io/kf-feast/feast-jc:"${FEAST_VERSION}" +wait_for_docker_image gcr.io/kf-feast/feast-jobcontroller:"${FEAST_VERSION}" wait_for_docker_image gcr.io/kf-feast/feast-serving:"${FEAST_VERSION}" wait_for_docker_image gcr.io/kf-feast/feast-jupyter:"${FEAST_VERSION}" @@ -66,11 +66,11 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett # Wait for Feast Core to be ready "${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120 -# Get Feast Job Coordinator container IP address -export FEAST_JC_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jc_1) +# Get Feast Job Controller container IP address +export FEAST_jobcontroller_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1) -# Wait for Feast Job Coordinator to be ready -"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JC_CONTAINER_IP_ADDRESS}:6570 --timeout=120 +# Wait for Feast Job Controller to be ready +"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_jobcontroller_CONTAINER_IP_ADDRESS}:6570 --timeout=120 # Get Feast Online Serving container IP address export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1) diff --git a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml index d47e7e8db9a..77fce72d292 100644 --- a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml +++ b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml @@ -16,7 +16,7 @@ feast-core: options: bootstrapServers: $feast_kafka_1_ip:31090 -feast-jc: +feast-jobcontroller: enabled: true gcpServiceAccount: enabled: true @@ -34,7 +34,7 @@ feast-jc: specsAckTopic: $SPECS_TOPIC-ack jobs: active_runner: dataflow - coordinator: + controller: consolidate-jobs-per-source: true jobSelector: application: feast diff --git a/infra/scripts/validate-version-consistency.sh b/infra/scripts/validate-version-consistency.sh index a3138ed8f62..f33cb8a56f4 100755 --- a/infra/scripts/validate-version-consistency.sh +++ b/infra/scripts/validate-version-consistency.sh @@ -41,9 +41,9 @@ declare -a files_to_validate_version=( "infra/charts/feast/charts/feast-jupyter/Chart.yaml,1,${FEAST_MASTER_VERSION}" "infra/charts/feast/charts/feast-serving/values.yaml,1,${FEAST_RELEASE_VERSION}" "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-jc/Chart.yaml,1,${FEAST_MASTER_VERSION}" - "infra/charts/feast/charts/feast-jc/values.yaml,1,${FEAST_RELEASE_VERSION}" - "infra/charts/feast/charts/feast-jc/README.md,1,${FEAST_RELEASE_VERSION}" + "infra/charts/feast/charts/feast-jobcontroller/Chart.yaml,1,${FEAST_MASTER_VERSION}" + "infra/charts/feast/charts/feast-jobcontroller/values.yaml,1,${FEAST_RELEASE_VERSION}" + "infra/charts/feast/charts/feast-jobcontroller/README.md,1,${FEAST_RELEASE_VERSION}" "infra/charts/feast/requirements.yaml,4,${FEAST_MASTER_VERSION}" "infra/charts/feast/requirements.lock,4,${FEAST_RELEASE_VERSION}" "infra/docker-compose/.env.sample,1,${FEAST_RELEASE_VERSION}" diff --git a/job-coordinator/pom.xml b/job-controller/pom.xml similarity index 98% rename from job-coordinator/pom.xml rename to job-controller/pom.xml index 5cebdd95e1f..b2fab14d84a 100644 --- a/job-coordinator/pom.xml +++ b/job-controller/pom.xml @@ -26,9 +26,9 @@ ${revision} - Feast Job Coordinator - Feature ingestion coordinator - feast-job-coordinator + Feast Job Controller + Feature ingestion controller + feast-job-controller diff --git a/job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java b/job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java similarity index 89% rename from job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java rename to job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java index 0fca1ff6eb7..e90e3e2caa8 100644 --- a/job-coordinator/src/main/java/feast/jc/JobCoordinatorApplication.java +++ b/job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc; +package feast.jobcontroller; -import feast.jc.config.FeastProperties; +import feast.jobcontroller.config.FeastProperties; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -35,8 +35,8 @@ }) @EnableConfigurationProperties(FeastProperties.class) @Slf4j -public class JobCoordinatorApplication { +public class JobControllerApplication { public static void main(String[] args) { - SpringApplication.run(JobCoordinatorApplication.class, args); + SpringApplication.run(JobControllerApplication.class, args); } } diff --git a/job-coordinator/src/main/java/feast/jc/config/FeastProperties.java b/job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/config/FeastProperties.java rename to job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java index 322692ac3e4..6b917888fe4 100644 --- a/job-coordinator/src/main/java/feast/jc/config/FeastProperties.java +++ b/job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.config; +package feast.jobcontroller.config; import feast.common.logging.config.LoggingProperties; import feast.common.validators.OneOfStrings; @@ -56,7 +56,7 @@ public FeastProperties(BuildProperties buildProperties) { /** Instantiates a new Feast properties. */ public FeastProperties() {} - /* Feast Job Coordinator Build Version */ + /* Feast Job Controller Build Version */ @NotBlank private String version = "unknown"; /* Feast Core Address */ @@ -89,16 +89,16 @@ public static class JobProperties { /* The active Apache Beam runner name. This name references one instance of the Runner class */ private String activeRunner; - /* Job Coordinator related properties */ - private CoordinatorProperties coordinator; + /* Job Controller related properties */ + private ControllerProperties controller; @Getter @Setter - public static class CoordinatorProperties { + public static class ControllerProperties { /* If true only one IngestionJob would be created per source with all subscribed stores in it */ private Boolean consolidateJobsPerSource = false; - /* Labels to identify jobs managed by this job coordinator */ + /* Labels to identify jobs managed by this job controller */ private Map jobSelector = new HashMap<>(); /* Selectors to define featureSets that are responsibility of current JobManager */ @@ -165,10 +165,10 @@ public static class Runner { /** * Gets the job runner type as an enum. * - * @return Returns the job runner type as {@link feast.jc.runner.Runner} + * @return Returns the job runner type as {@link feast.jobcontroller.runner.Runner} */ - public feast.jc.runner.Runner getType() { - return feast.jc.runner.Runner.fromString(type); + public feast.jobcontroller.runner.Runner getType() { + return feast.jobcontroller.runner.Runner.fromString(type); } } diff --git a/job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java b/job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java similarity index 99% rename from job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java rename to job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java index 858047a8f09..c7eb63f9842 100644 --- a/job-coordinator/src/main/java/feast/jc/config/FeatureStreamConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.config; +package feast.jobcontroller.config; import feast.common.util.KafkaSerialization; import feast.proto.core.FeatureSetProto; diff --git a/job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java b/job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java similarity index 89% rename from job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java rename to job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java index 48e812dc8a3..5d13d699340 100644 --- a/job-coordinator/src/main/java/feast/jc/config/JobCoordinatorConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java @@ -14,19 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.config; +package feast.jobcontroller.config; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import feast.jc.dao.JobRepository; -import feast.jc.runner.ConsolidatedJobStrategy; -import feast.jc.runner.JobGroupingStrategy; -import feast.jc.runner.JobManager; -import feast.jc.runner.JobPerStoreStrategy; -import feast.jc.runner.dataflow.DataflowJobManager; -import feast.jc.runner.direct.DirectJobRegistry; -import feast.jc.runner.direct.DirectRunnerJobManager; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.runner.ConsolidatedJobStrategy; +import feast.jobcontroller.runner.JobGroupingStrategy; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.JobPerStoreStrategy; +import feast.jobcontroller.runner.dataflow.DataflowJobManager; +import feast.jobcontroller.runner.direct.DirectJobRegistry; +import feast.jobcontroller.runner.direct.DirectRunnerJobManager; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; @@ -45,7 +45,7 @@ /** Beans for job management */ @Slf4j @Configuration -public class JobCoordinatorConfig { +public class JobControllerConfig { private final Gson gson = new Gson(); /** @@ -85,7 +85,7 @@ public IngestionJobProto.SpecsStreamingUpdateConfig createSpecsStreamingUpdateCo public JobGroupingStrategy getJobGroupingStrategy( FeastProperties feastProperties, JobRepository jobRepository) { Boolean shouldConsolidateJobs = - feastProperties.getJobs().getCoordinator().getConsolidateJobsPerSource(); + feastProperties.getJobs().getController().getConsolidateJobsPerSource(); if (shouldConsolidateJobs) { return new ConsolidatedJobStrategy(jobRepository); } else { @@ -121,7 +121,7 @@ public JobManager getJobManager( dataflowRunnerConfigOptions.build(), metrics, specsStreamingUpdateConfig, - jobProperties.getCoordinator().getJobSelector()); + jobProperties.getController().getJobSelector()); case DIRECT: DirectRunnerConfigOptions.Builder directRunnerConfigOptions = DirectRunnerConfigOptions.newBuilder(); diff --git a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java b/job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java rename to job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java index 76e960a50b3..97d21cf3ace 100644 --- a/job-coordinator/src/main/java/feast/jc/dao/InMemoryJobRepository.java +++ b/job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.dao; +package feast.jobcontroller.dao; import com.google.common.collect.Lists; import feast.common.models.FeatureSetReference; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import feast.proto.core.SourceProto; import java.util.*; import java.util.function.Predicate; diff --git a/job-coordinator/src/main/java/feast/jc/dao/JobRepository.java b/job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java similarity index 92% rename from job-coordinator/src/main/java/feast/jc/dao/JobRepository.java rename to job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java index e9737d3863e..c70716b1696 100644 --- a/job-coordinator/src/main/java/feast/jc/dao/JobRepository.java +++ b/job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.dao; +package feast.jobcontroller.dao; import feast.common.models.FeatureSetReference; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; import feast.proto.core.SourceProto; import java.util.Collection; import java.util.List; diff --git a/job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java b/job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java rename to job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java index 61f918a13f8..035640c323c 100644 --- a/job-coordinator/src/main/java/feast/jc/exception/JobExecutionException.java +++ b/job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.exception; +package feast.jobcontroller.exception; /** Exception thrown when a request for job execution fails. */ public class JobExecutionException extends RuntimeException { diff --git a/job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java b/job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java rename to job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java index 511046b3112..1a8ec749af2 100644 --- a/job-coordinator/src/main/java/feast/jc/exception/JobMonitoringException.java +++ b/job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.exception; +package feast.jobcontroller.exception; /** Exception thrown when error happen during job monitoring. */ public class JobMonitoringException extends RuntimeException { diff --git a/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java b/job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java rename to job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java index 567170753fb..607e6fbe7f7 100644 --- a/job-coordinator/src/main/java/feast/jc/grpc/CoreServiceImpl.java +++ b/job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.grpc; +package feast.jobcontroller.grpc; import com.google.api.gax.rpc.InvalidArgumentException; import feast.common.interceptors.GrpcMessageInterceptor; -import feast.jc.service.JobService; +import feast.jobcontroller.service.JobService; import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.JobCoordinatorServiceGrpc.JobCoordinatorServiceImplBase; +import feast.proto.core.JobControllerServiceGrpc.JobControllerServiceImplBase; import io.grpc.Status; import io.grpc.stub.StreamObserver; import java.util.NoSuchElementException; @@ -31,7 +31,7 @@ /** Implementation of the feast core GRPC service. */ @Slf4j @GrpcService(interceptors = {GrpcMessageInterceptor.class}) -public class CoreServiceImpl extends JobCoordinatorServiceImplBase { +public class CoreServiceImpl extends JobControllerServiceImplBase { private JobService jobService; diff --git a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java b/job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java similarity index 98% rename from job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java rename to job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java index 3b1f067e668..dfc69a37a59 100644 --- a/job-coordinator/src/main/java/feast/jc/grpc/HealthServiceImpl.java +++ b/job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.grpc; +package feast.jobcontroller.grpc; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceProto; diff --git a/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java b/job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java similarity index 98% rename from job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java rename to job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java index 8ab0db966d9..e7300f3be2d 100644 --- a/job-coordinator/src/main/java/feast/jc/model/FeatureSetDeliveryStatus.java +++ b/job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.model; +package feast.jobcontroller.model; import com.google.common.base.Objects; import feast.common.models.FeatureSetReference; diff --git a/job-coordinator/src/main/java/feast/jc/model/Job.java b/job-controller/src/main/java/feast/jobcontroller/model/Job.java similarity index 99% rename from job-coordinator/src/main/java/feast/jc/model/Job.java rename to job-controller/src/main/java/feast/jobcontroller/model/Job.java index 03fbb8d476e..64193998048 100644 --- a/job-coordinator/src/main/java/feast/jc/model/Job.java +++ b/job-controller/src/main/java/feast/jobcontroller/model/Job.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.model; +package feast.jobcontroller.model; import com.google.auto.value.AutoValue; import feast.common.models.FeatureSetReference; diff --git a/job-coordinator/src/main/java/feast/jc/model/JobStatus.java b/job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java similarity index 99% rename from job-coordinator/src/main/java/feast/jc/model/JobStatus.java rename to job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java index 25a65892ea9..3e8824c743d 100644 --- a/job-coordinator/src/main/java/feast/jc/model/JobStatus.java +++ b/job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.model; +package feast.jobcontroller.model; import feast.proto.core.IngestionJobProto.IngestionJobStatus; import java.util.Map; diff --git a/job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java rename to job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java index 1f9e302e312..567944cab4e 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/ConsolidatedJobStrategy.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; -import feast.jc.dao.JobRepository; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.time.Instant; diff --git a/job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java rename to job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java index 34d2df3c7a6..0a672808ab5 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/JobGroupingStrategy.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; -import feast.jc.model.Job; +import feast.jobcontroller.model.Job; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.util.Map; diff --git a/job-coordinator/src/main/java/feast/jc/runner/JobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java similarity index 94% rename from job-coordinator/src/main/java/feast/jc/runner/JobManager.java rename to job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java index 06891761260..c9cbaf42a0a 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/JobManager.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; import java.util.List; public interface JobManager { diff --git a/job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java rename to job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java index 17d6874bf93..1c63bdac4b3 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/JobPerStoreStrategy.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; import com.google.common.collect.Lists; -import feast.jc.dao.JobRepository; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import java.time.Instant; diff --git a/job-coordinator/src/main/java/feast/jc/runner/Runner.java b/job-controller/src/main/java/feast/jobcontroller/runner/Runner.java similarity index 97% rename from job-coordinator/src/main/java/feast/jc/runner/Runner.java rename to job-controller/src/main/java/feast/jobcontroller/runner/Runner.java index 6a5f3449127..8db14b5d2e4 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/Runner.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/Runner.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; import java.util.NoSuchElementException; diff --git a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java similarity index 97% rename from job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java rename to job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java index 215156e2bdf..034d3b0f8f4 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobManager.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; import static feast.ingestion.utils.SpecUtil.parseSourceJson; import static feast.ingestion.utils.SpecUtil.parseStoreJsonList; -import static feast.jc.util.PipelineUtil.detectClassPathResourcesToStage; +import static feast.jobcontroller.util.PipelineUtil.detectClassPathResourcesToStage; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; @@ -32,12 +32,12 @@ import feast.common.models.FeatureSetReference; import feast.ingestion.ImportJob; import feast.ingestion.options.ImportOptions; -import feast.jc.config.FeastProperties.MetricsProperties; -import feast.jc.exception.JobExecutionException; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; -import feast.jc.runner.Runner; +import feast.jobcontroller.config.FeastProperties.MetricsProperties; +import feast.jobcontroller.exception.JobExecutionException; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.Runner; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import feast.proto.core.SourceProto; diff --git a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java rename to job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java index 49992208828..e66ed701e54 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobState.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; public enum DataflowJobState { JOB_STATE_UNKNOWN, diff --git a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java similarity index 93% rename from job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java rename to job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java index 0f46652f895..fc0ad632b69 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobStateMapper.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; -import static feast.jc.runner.dataflow.DataflowJobState.*; +import static feast.jobcontroller.runner.dataflow.DataflowJobState.*; -import feast.jc.model.JobStatus; +import feast.jobcontroller.model.JobStatus; import java.util.HashMap; import java.util.Map; diff --git a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java similarity index 94% rename from job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java rename to job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java index 578e7c7cd46..1f9c75ecc55 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowJobType.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; public enum DataflowJobType { JOB_TYPE_BATCH, diff --git a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java similarity index 97% rename from job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java rename to job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java index d723ca2b128..ec7d0eec8f5 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/dataflow/DataflowRunnerConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; -import feast.jc.runner.option.RunnerConfig; +import feast.jobcontroller.runner.option.RunnerConfig; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import java.util.Map; import java.util.Set; diff --git a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java rename to job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java index 600572581ac..50eafc3aee0 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJob.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; import java.io.IOException; import lombok.AllArgsConstructor; diff --git a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java similarity index 97% rename from job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java rename to job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java index da3567f0d84..f3cd2a4485f 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobRegistry.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; import com.google.common.base.Strings; import java.io.IOException; diff --git a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java similarity index 94% rename from job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java rename to job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java index dd649a4d08f..016484ff667 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectJobStateMapper.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; -import feast.jc.model.JobStatus; +import feast.jobcontroller.model.JobStatus; import java.util.HashMap; import java.util.Map; import org.apache.beam.sdk.PipelineResult.State; diff --git a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java similarity index 93% rename from job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java rename to job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java index 6e53085a588..d4b518d26e6 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; -import feast.jc.runner.option.RunnerConfig; +import feast.jobcontroller.runner.option.RunnerConfig; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; public class DirectRunnerConfig extends RunnerConfig { diff --git a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java rename to job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java index 4e81394c24e..358d2c29e54 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/direct/DirectRunnerJobManager.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java @@ -14,19 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; import com.google.common.base.Strings; import com.google.protobuf.util.JsonFormat; import feast.common.models.FeatureSetReference; import feast.ingestion.ImportJob; import feast.ingestion.options.ImportOptions; -import feast.jc.config.FeastProperties.MetricsProperties; -import feast.jc.exception.JobExecutionException; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; -import feast.jc.runner.Runner; +import feast.jobcontroller.config.FeastProperties.MetricsProperties; +import feast.jobcontroller.exception.JobExecutionException; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.Runner; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import feast.proto.core.SourceProto; diff --git a/job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java b/job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java similarity index 97% rename from job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java rename to job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java index 3366527c55d..426bd372124 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/option/FeatureSetJsonByteConverter.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.option; +package feast.jobcontroller.runner.option; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; diff --git a/job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java similarity index 92% rename from job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java rename to job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java index 36cd67aba52..dba49a5cd8d 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/option/RunnerConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.option; +package feast.jobcontroller.runner.option; -import feast.jc.util.TypeConversion; +import feast.jobcontroller.util.TypeConversion; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -24,7 +24,7 @@ /** * Value class containing the application-default configuration for a runner. When a job is started - * by JC, all fields in the object will be converted into --key=value args to seed the beam pipeline + * by jobcontroller, all fields in the object will be converted into --key=value args to seed the beam pipeline * options. */ public abstract class RunnerConfig { diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java similarity index 92% rename from job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java index d3cef51862a..cadadc0de8c 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/CreateJobTask.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java similarity index 92% rename from job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java index 0fa2b72cdda..294e078e58f 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/JobTask.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import java.util.concurrent.Callable; import lombok.Getter; import lombok.Setter; diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java index bb0148cd8cf..f913b9b56c8 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/JobTasks.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; /** Enum listing of the available Job Tasks to perform on Jobs */ public enum JobTasks { diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java similarity index 90% rename from job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java index 1f9ebac2a4a..39183d24aa6 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/RestartJobTask.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java similarity index 89% rename from job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java index b5df9f5d271..ade6ebc9599 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/TerminateJobTask.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; import feast.common.logging.AuditLogger; import feast.common.logging.entry.LogResource.ResourceType; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; diff --git a/job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java similarity index 87% rename from job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java rename to job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java index fd27757b13b..9a5f4ade8b1 100644 --- a/job-coordinator/src/main/java/feast/jc/runner/task/UpdateJobStatusTask.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; /** * Task that retrieves status from {@link JobManager} on given {@link Job} and update the job diff --git a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java similarity index 95% rename from job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java rename to job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java index fc1216279e5..7cd896b3581 100644 --- a/job-coordinator/src/main/java/feast/jc/service/JobCoordinatorService.java +++ b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java @@ -14,24 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import static feast.common.models.Store.isSubscribedToFeatureSet; import com.google.common.collect.Sets; import feast.common.models.FeatureSetReference; -import feast.jc.config.FeastProperties; -import feast.jc.config.FeastProperties.JobProperties; -import feast.jc.dao.JobRepository; -import feast.jc.model.FeatureSetDeliveryStatus; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobGroupingStrategy; -import feast.jc.runner.JobManager; -import feast.jc.runner.task.CreateJobTask; -import feast.jc.runner.task.JobTask; -import feast.jc.runner.task.TerminateJobTask; -import feast.jc.runner.task.UpdateJobStatusTask; +import feast.jobcontroller.config.FeastProperties; +import feast.jobcontroller.config.FeastProperties.JobProperties; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.FeatureSetDeliveryStatus; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobGroupingStrategy; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.task.CreateJobTask; +import feast.jobcontroller.runner.task.JobTask; +import feast.jobcontroller.runner.task.TerminateJobTask; +import feast.jobcontroller.runner.task.UpdateJobStatusTask; import feast.proto.core.*; import feast.proto.core.CoreServiceProto.ListStoresRequest.Filter; import feast.proto.core.CoreServiceProto.ListStoresResponse; @@ -57,7 +57,7 @@ @Slf4j @Service @ConditionalOnProperty("feast.jobs.enabled") -public class JobCoordinatorService { +public class JobControllerService { private final int SPEC_PUBLISHING_TIMEOUT_SECONDS = 5; public static final String VERSION_LABEL = "feast_version"; @@ -74,7 +74,7 @@ public class JobCoordinatorService { private final String currentVersion; @Autowired - public JobCoordinatorService( + public JobControllerService( JobRepository jobRepository, CoreServiceGrpc.CoreServiceBlockingStub specService, JobManager jobManager, @@ -88,12 +88,12 @@ public JobCoordinatorService( this.specPublisher = specPublisher; this.groupingStrategy = groupingStrategy; this.featureSetSubscriptions = - feastProperties.getJobs().getCoordinator().getFeatureSetSelector().stream() - .map(JobProperties.CoordinatorProperties.FeatureSetSelector::toSubscription) + feastProperties.getJobs().getController().getFeatureSetSelector().stream() + .map(JobProperties.ControllerProperties.FeatureSetSelector::toSubscription) .collect(Collectors.toList()); - this.whitelistedStores = feastProperties.getJobs().getCoordinator().getWhitelistedStores(); + this.whitelistedStores = feastProperties.getJobs().getController().getWhitelistedStores(); this.currentVersion = feastProperties.getVersion(); - this.jobLabels = new HashMap<>(feastProperties.getJobs().getCoordinator().getJobSelector()); + this.jobLabels = new HashMap<>(feastProperties.getJobs().getController().getJobSelector()); this.jobLabels.put(VERSION_LABEL, getCurrentFeastVersion()); } diff --git a/job-coordinator/src/main/java/feast/jc/service/JobService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobService.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/service/JobService.java rename to job-controller/src/main/java/feast/jobcontroller/service/JobService.java index 13e0ac7ae1a..7b99fbf733d 100644 --- a/job-coordinator/src/main/java/feast/jc/service/JobService.java +++ b/job-controller/src/main/java/feast/jobcontroller/service/JobService.java @@ -14,15 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import feast.common.models.FeatureSetReference; -import feast.jc.dao.JobRepository; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; -import feast.jc.runner.task.RestartJobTask; -import feast.jc.runner.task.TerminateJobTask; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.task.RestartJobTask; +import feast.jobcontroller.runner.task.TerminateJobTask; import feast.proto.core.CoreServiceProto.*; import feast.proto.core.FeatureSetReferenceProto; import feast.proto.core.IngestionJobProto; diff --git a/job-coordinator/src/main/java/feast/jc/util/PackageUtil.java b/job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java similarity index 99% rename from job-coordinator/src/main/java/feast/jc/util/PackageUtil.java rename to job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java index 2726e38d0e0..3eb4673413c 100644 --- a/job-coordinator/src/main/java/feast/jc/util/PackageUtil.java +++ b/job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.util; +package feast.jobcontroller.util; import java.io.File; import java.io.FileOutputStream; diff --git a/job-coordinator/src/main/java/feast/jc/util/PipelineUtil.java b/job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java similarity index 96% rename from job-coordinator/src/main/java/feast/jc/util/PipelineUtil.java rename to job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java index aca7e62c2a5..2cec65062a4 100644 --- a/job-coordinator/src/main/java/feast/jc/util/PipelineUtil.java +++ b/job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.util; +package feast.jobcontroller.util; -import static feast.jc.util.PackageUtil.resolveSpringBootPackageClasspath; +import static feast.jobcontroller.util.PackageUtil.resolveSpringBootPackageClasspath; import java.io.File; import java.io.IOException; diff --git a/job-coordinator/src/main/java/feast/jc/util/TypeConversion.java b/job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java similarity index 98% rename from job-coordinator/src/main/java/feast/jc/util/TypeConversion.java rename to job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java index f50b85f3f76..92b3c8dd6f7 100644 --- a/job-coordinator/src/main/java/feast/jc/util/TypeConversion.java +++ b/job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.util; +package feast.jobcontroller.util; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; diff --git a/job-coordinator/src/main/resources/application.yml b/job-controller/src/main/resources/application.yml similarity index 99% rename from job-coordinator/src/main/resources/application.yml rename to job-controller/src/main/resources/application.yml index ef15ddc6c00..909ba1b42b4 100644 --- a/job-coordinator/src/main/resources/application.yml +++ b/job-controller/src/main/resources/application.yml @@ -68,7 +68,7 @@ feast: # Port of the metrics sink. port: 9125 - coordinator: + controller: # if true one job per source with many stores would be created # if false one job per source-store pair would be created consolidate-jobs-per-source: false diff --git a/job-coordinator/src/main/resources/banner.txt b/job-controller/src/main/resources/banner.txt similarity index 100% rename from job-coordinator/src/main/resources/banner.txt rename to job-controller/src/main/resources/banner.txt diff --git a/job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java b/job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java similarity index 97% rename from job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java rename to job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java index 90ea68a6c05..884f9b73ec7 100644 --- a/job-coordinator/src/test/java/feast/jc/model/JobStatusTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.model; +package feast.jobcontroller.model; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; diff --git a/job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java similarity index 97% rename from job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java index d66230e67e8..98c8da3798f 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/RunnerTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner; +package feast.jobcontroller.runner; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; diff --git a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java similarity index 98% rename from job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java index d422c52a92a..4451bec142f 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobManagerTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -31,9 +31,9 @@ import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; import feast.ingestion.options.ImportOptions; -import feast.jc.config.FeastProperties; -import feast.jc.exception.JobExecutionException; -import feast.jc.model.Job; +import feast.jobcontroller.config.FeastProperties; +import feast.jobcontroller.exception.JobExecutionException; +import feast.jobcontroller.model.Job; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions.Builder; diff --git a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java similarity index 95% rename from job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java index ff1783fe0a2..62d1d6e59f7 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowJobStateMapperTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java similarity index 99% rename from job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java index bd49dc82d7a..1eeada75753 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/dataflow/DataflowRunnerConfigTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.dataflow; +package feast.jobcontroller.runner.dataflow; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; diff --git a/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java similarity index 97% rename from job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java index 1678e22c980..60f1c7fe695 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerConfigTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; diff --git a/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java similarity index 97% rename from job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java index 6fe573204c6..60bbd927c5c 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/direct/DirectRunnerJobManagerTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.direct; +package feast.jobcontroller.runner.direct; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -27,9 +27,9 @@ import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; import feast.ingestion.options.ImportOptions; -import feast.jc.config.FeastProperties.MetricsProperties; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; +import feast.jobcontroller.config.FeastProperties.MetricsProperties; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; import feast.proto.core.IngestionJobProto; import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; import feast.proto.core.SourceProto; diff --git a/job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java similarity index 98% rename from job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java index ca2ba5f42c7..b319a649e85 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/option/FeatureSetJsonByteConverterTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.option; +package feast.jobcontroller.runner.option; import static org.junit.Assert.assertEquals; diff --git a/job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java similarity index 95% rename from job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java rename to job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java index 729805814aa..4e8aaa4459a 100644 --- a/job-coordinator/src/test/java/feast/jc/runner/task/JobTasksTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.runner.task; +package feast.jobcontroller.runner.task; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; @@ -24,10 +24,10 @@ import com.google.common.collect.ImmutableMap; import feast.common.util.TestUtil; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; -import feast.jc.runner.Runner; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.Runner; import feast.proto.core.SourceProto; import feast.proto.core.SourceProto.KafkaSourceConfig; import feast.proto.core.SourceProto.SourceType; diff --git a/job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java b/job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java similarity index 89% rename from job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java rename to job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java index 27ca84ecaed..22c56fc6fc0 100644 --- a/job-coordinator/src/test/java/feast/jc/service/FakeJobManager.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import com.google.common.collect.Lists; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; -import feast.jc.runner.Runner; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.Runner; import java.util.*; public class FakeJobManager implements JobManager { diff --git a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java similarity index 93% rename from job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java rename to job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java index d367ec43652..ead71d1f67a 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorIT.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.*; @@ -32,11 +32,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.common.it.*; import feast.common.util.KafkaSerialization; -import feast.jc.config.FeastProperties; -import feast.jc.dao.JobRepository; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.config.FeastProperties; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import feast.proto.core.*; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; @@ -67,14 +67,14 @@ "feast.jobs.enabled=true", "feast.jobs.polling_interval_milliseconds=1000", "feast.stream.specsOptions.notifyIntervalMilliseconds=1000", - "feast.jobs.coordinator.consolidate-jobs-per-source=true", - "feast.jobs.coordinator.feature-set-selector[0].name=test", - "feast.jobs.coordinator.feature-set-selector[0].project=default", - "feast.jobs.coordinator.whitelisted-stores[0]=test-store", - "feast.jobs.coordinator.whitelisted-stores[1]=new-store", + "feast.jobs.controller.consolidate-jobs-per-source=true", + "feast.jobs.controller.feature-set-selector[0].name=test", + "feast.jobs.controller.feature-set-selector[0].project=default", + "feast.jobs.controller.whitelisted-stores[0]=test-store", + "feast.jobs.controller.whitelisted-stores[1]=new-store", "feast.version=1.0.0" }) -public class JobCoordinatorIT extends BaseIT { +public class JobControllerIT extends BaseIT { @Autowired private FakeJobManager jobManager; @Autowired private JobRepository jobRepository; @@ -111,7 +111,7 @@ public static void globalSetUp(@Value("${grpc.server.port}") int localPort) { apiClient = new SimpleJCClient( - JobCoordinatorServiceGrpc.newBlockingStub( + JobControllerServiceGrpc.newBlockingStub( ManagedChannelBuilder.forAddress("localhost", localPort).usePlaintext().build())); } @@ -241,7 +241,7 @@ public void shouldRestartJobWithOldVersion() { ImmutableMap.of( DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) .setId("some-running-id") - .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "0-9-9")) + .setLabels(ImmutableMap.of(JobControllerService.VERSION_LABEL, "0-9-9")) .build(); jobManager.startJob(job); @@ -252,7 +252,7 @@ public void shouldRestartJobWithOldVersion() { Job replacement = jobRepository.findByStatus(JobStatus.RUNNING).get(0); assertThat(replacement.getSource(), equalTo(job.getSource())); assertThat(replacement.getStores(), equalTo(job.getStores())); - assertThat(replacement.getLabels(), hasEntry(JobCoordinatorService.VERSION_LABEL, "1-0-0")); + assertThat(replacement.getLabels(), hasEntry(JobControllerService.VERSION_LABEL, "1-0-0")); } @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -279,7 +279,7 @@ public void shouldSendNewSpec() { ImmutableMap.of( DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) .setId("some-running-id") - .setLabels(ImmutableMap.of(JobCoordinatorService.VERSION_LABEL, "1-0-0")) + .setLabels(ImmutableMap.of(JobControllerService.VERSION_LABEL, "1-0-0")) .build(); jobManager.startJob(job); diff --git a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java similarity index 85% rename from job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java rename to job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java index 45895b249c5..8abdb62a2db 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobCoordinatorServiceTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -31,18 +31,18 @@ import com.google.common.collect.*; import com.google.protobuf.InvalidProtocolBufferException; import feast.common.it.DataGenerator; -import feast.jc.config.FeastProperties; -import feast.jc.config.FeastProperties.JobProperties; -import feast.jc.dao.InMemoryJobRepository; -import feast.jc.dao.JobRepository; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.ConsolidatedJobStrategy; -import feast.jc.runner.JobManager; -import feast.jc.runner.JobPerStoreStrategy; -import feast.jc.runner.task.CreateJobTask; -import feast.jc.runner.task.JobTask; -import feast.jc.runner.task.UpdateJobStatusTask; +import feast.jobcontroller.config.FeastProperties; +import feast.jobcontroller.config.FeastProperties.JobProperties; +import feast.jobcontroller.dao.InMemoryJobRepository; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.ConsolidatedJobStrategy; +import feast.jobcontroller.runner.JobManager; +import feast.jobcontroller.runner.JobPerStoreStrategy; +import feast.jobcontroller.runner.task.CreateJobTask; +import feast.jobcontroller.runner.task.JobTask; +import feast.jobcontroller.runner.task.UpdateJobStatusTask; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceProto; import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest.Filter; @@ -60,15 +60,15 @@ import org.mockito.Mock; import org.springframework.kafka.core.KafkaTemplate; -public class JobCoordinatorServiceTest { +public class JobControllerServiceTest { JobRepository jobRepository; @Mock CoreServiceGrpc.CoreServiceBlockingStub specService; private FeastProperties feastProperties; - private JobCoordinatorService jcsWithConsolidation; - private JobCoordinatorService jcsWithJobPerStore; + private JobControllerService jobcontrollersWithConsolidation; + private JobControllerService jobcontrollersWithJobPerStore; @BeforeEach public void setUp() { @@ -77,19 +77,19 @@ public void setUp() { JobProperties jobProperties = new JobProperties(); jobProperties.setJobUpdateTimeoutSeconds(5); - FeastProperties.JobProperties.CoordinatorProperties.FeatureSetSelector selector = - new FeastProperties.JobProperties.CoordinatorProperties.FeatureSetSelector(); + FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector selector = + new FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector(); selector.setName("fs*"); selector.setProject("*"); - FeastProperties.JobProperties.CoordinatorProperties coordinatorProperties = - new FeastProperties.JobProperties.CoordinatorProperties(); - coordinatorProperties.setFeatureSetSelector(ImmutableList.of(selector)); - coordinatorProperties.setWhitelistedStores( + FeastProperties.JobProperties.ControllerProperties controllerProperties = + new FeastProperties.JobProperties.ControllerProperties(); + controllerProperties.setFeatureSetSelector(ImmutableList.of(selector)); + controllerProperties.setWhitelistedStores( ImmutableList.of("test-store", "test", "test-1", "test-2", "normal-store")); - coordinatorProperties.setJobSelector(ImmutableMap.of("application", "feast")); + controllerProperties.setJobSelector(ImmutableMap.of("application", "feast")); - jobProperties.setCoordinator(coordinatorProperties); + jobProperties.setController(controllerProperties); feastProperties.setJobs(jobProperties); feastProperties.setVersion("1.0.0"); @@ -98,8 +98,8 @@ public void setUp() { when(jobManager.listRunningJobs()).thenReturn(Collections.emptyList()); jobRepository = new InMemoryJobRepository(jobManager); - jcsWithConsolidation = - new JobCoordinatorService( + jobcontrollersWithConsolidation = + new JobControllerService( jobRepository, specService, jobManager, @@ -107,8 +107,8 @@ public void setUp() { new ConsolidatedJobStrategy(jobRepository), mock(KafkaTemplate.class)); - jcsWithJobPerStore = - new JobCoordinatorService( + jobcontrollersWithJobPerStore = + new JobControllerService( jobRepository, specService, jobManager, @@ -122,7 +122,7 @@ public void shouldDoNothingIfNoStoresFound() { when(specService.listStores(any())).thenReturn(ListStoresResponse.newBuilder().build()); List jobTasks = - jcsWithConsolidation.makeJobUpdateTasks(jcsWithConsolidation.getSourceToStoreMappings()); + jobcontrollersWithConsolidation.makeJobUpdateTasks(jobcontrollersWithConsolidation.getSourceToStoreMappings()); assertThat(jobTasks, hasSize(0)); } @@ -140,7 +140,7 @@ public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocol .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List jobTasks = - jcsWithConsolidation.makeJobUpdateTasks(jcsWithConsolidation.getSourceToStoreMappings()); + jobcontrollersWithConsolidation.makeJobUpdateTasks(jobcontrollersWithConsolidation.getSourceToStoreMappings()); assertThat(jobTasks, hasSize(0)); } @@ -171,7 +171,7 @@ public void shouldGroupJobsBySource() { .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); ArrayList>> pairs = - Lists.newArrayList(jcsWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, hasSize(2)); assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store)))); @@ -219,7 +219,7 @@ public void shouldUseStoreSubscriptionToMapStore() { .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); ArrayList>> pairs = - Lists.newArrayList(jcsWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, hasSize(2)); assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store1)))); @@ -243,7 +243,7 @@ public void shouldCheckStatusOfAbortingJob() { jobRepository.add(job); List tasks = - jcsWithConsolidation.makeJobUpdateTasks( + jobcontrollersWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CheckStatus is expected", tasks.get(0) instanceof UpdateJobStatusTask); @@ -262,7 +262,7 @@ public void shouldUpgradeJobWhenNeeded() { jobRepository.add(job); List tasks = - jcsWithConsolidation.makeJobUpdateTasks( + jobcontrollersWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); @@ -281,7 +281,7 @@ public void shouldCreateJobIfNoRunning() { .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List tasks = - jcsWithConsolidation.makeJobUpdateTasks( + jobcontrollersWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); @@ -312,7 +312,7 @@ public void shouldCreateJobPerStore() throws InvalidProtocolBufferException { .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); List jobTasks = - jcsWithJobPerStore.makeJobUpdateTasks(jcsWithJobPerStore.getSourceToStoreMappings()); + jobcontrollersWithJobPerStore.makeJobUpdateTasks(jobcontrollersWithJobPerStore.getSourceToStoreMappings()); int hash = Objects.hash( @@ -358,7 +358,7 @@ public void shouldCloneRunningJobOnUpgrade() throws InvalidProtocolBufferExcepti jobRepository.add(existingJob); List jobTasks = - jcsWithConsolidation.makeJobUpdateTasks( + jobcontrollersWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store1, store2)))); assertThat(jobTasks, hasSize(1)); @@ -384,7 +384,7 @@ public void shouldSelectOnlyFeatureSetsThatJobManagerSubscribedTo() { .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2, featureSet3)) .build()); - List featureSetsForStore = jcsWithConsolidation.getFeatureSetsForStore(store); + List featureSetsForStore = jobcontrollersWithConsolidation.getFeatureSetsForStore(store); assertThat(featureSetsForStore, containsInAnyOrder(featureSet1, featureSet2)); } @@ -426,7 +426,7 @@ public void shouldSelectOnlyStoresThatWhitelisted() { .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet2).build()); ArrayList>> pairs = - Lists.newArrayList(jcsWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, containsInAnyOrder(Pair.of(source1, ImmutableSet.of(store1)))); } diff --git a/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java similarity index 94% rename from job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java rename to job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java index 10c402fc115..3a6e2a23966 100644 --- a/job-coordinator/src/test/java/feast/jc/service/JobServiceIT.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.jc.service; +package feast.jobcontroller.service; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -26,14 +26,14 @@ import feast.common.it.BaseIT; import feast.common.it.DataGenerator; import feast.common.models.FeatureSetReference; -import feast.jc.dao.JobRepository; -import feast.jc.model.FeatureSetDeliveryStatus; -import feast.jc.model.Job; -import feast.jc.model.JobStatus; -import feast.jc.runner.JobManager; +import feast.jobcontroller.dao.JobRepository; +import feast.jobcontroller.model.FeatureSetDeliveryStatus; +import feast.jobcontroller.model.Job; +import feast.jobcontroller.model.JobStatus; +import feast.jobcontroller.runner.JobManager; import feast.proto.core.CoreServiceProto; import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.core.JobCoordinatorServiceGrpc; +import feast.proto.core.JobControllerServiceGrpc; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; @@ -50,7 +50,7 @@ public class JobServiceIT extends BaseIT { @Autowired private JobRepository jobRepository; - static JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub; + static JobControllerServiceGrpc.JobControllerServiceBlockingStub stub; Job job; @@ -58,7 +58,7 @@ public class JobServiceIT extends BaseIT { public static void globalSetUp(@Value("${grpc.server.port}") int port) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = JobCoordinatorServiceGrpc.newBlockingStub(channel); + stub = JobControllerServiceGrpc.newBlockingStub(channel); } private Job createJobWithId(String jobId) { diff --git a/job-coordinator/src/test/resources/application-it-core.yml b/job-controller/src/test/resources/application-it-core.yml similarity index 100% rename from job-coordinator/src/test/resources/application-it-core.yml rename to job-controller/src/test/resources/application-it-core.yml diff --git a/job-coordinator/src/test/resources/application-it.properties b/job-controller/src/test/resources/application-it.properties similarity index 100% rename from job-coordinator/src/test/resources/application-it.properties rename to job-controller/src/test/resources/application-it.properties diff --git a/job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/job-controller/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker similarity index 100% rename from job-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker rename to job-controller/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/pom.xml b/pom.xml index 56d65ec708a..bc31a95425f 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ docs/coverage/java common auth - job-coordinator + job-controller common-test diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index 658b4940b52..59a50b823f0 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -91,12 +91,12 @@ service CoreService { // Lists all projects active projects. rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse); - // Internal API for Job Coordinator to update featureSet's status once responsible ingestion job is running + // Internal API for Job Controller to update featureSet's status once responsible ingestion job is running rpc UpdateFeatureSetStatus (UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); } -service JobCoordinatorService { +service JobControllerService { // List Ingestion Jobs given an optional filter. // Returns allow ingestions matching the given request filter. // Returns all ingestion jobs if no filter is provided. diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index debaf6f13c8..aaea3ff8b45 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -23,7 +23,7 @@ from feast.client import Client from feast.config import Config -from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient +from feast.contrib.job_controller.client import Client as JCClient from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.feature_set import FeatureSet, FeatureSetRef from feast.loaders.yaml import yaml_loader @@ -352,7 +352,7 @@ def ingest_job_list(job_id, feature_set_ref, store_name): feature_set_ref = FeatureSetRef.from_str(feature_set_ref) # pull & render ingestion jobs as a table - feast_client = JobCoordinatorClient() + feast_client = JCClient() table = [] for ingest_job in feast_client.list_ingest_jobs( job_id=job_id, feature_set_ref=feature_set_ref, store_name=store_name @@ -371,7 +371,7 @@ def ingest_job_describe(job_id: str): Describe the ingestion job with the given id. """ # find ingestion job for id - feast_client = JobCoordinatorClient() + feast_client = JCClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") @@ -400,7 +400,7 @@ def ingest_job_stop(wait: bool, timeout: int, job_id: str): Stop ingestion job for id. """ # find ingestion job for id - feast_client = JobCoordinatorClient() + feast_client = JCClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") @@ -422,7 +422,7 @@ def ingest_job_restart(job_id: str): Waits for the job to fully restart. """ # find ingestion job for id - feast_client = JobCoordinatorClient() + feast_client = JCClient() jobs = feast_client.list_ingest_jobs(job_id=job_id) if len(jobs) < 1: print(f"Ingestion Job with id {job_id} could not be found") diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 358c70b6661..7ed0d273fe3 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -45,7 +45,7 @@ class AuthProvider(Enum): CONFIG_ENABLE_AUTH_KEY = "enable_auth" CONFIG_ENABLE_AUTH_TOKEN_KEY = "auth_token" CONFIG_CORE_SERVER_SSL_CERT_KEY = "core_server_ssl_cert" -CONFIG_JC_SERVER_KEY = "jc_url" +CONFIG_JOB_CONTROLLER_SERVER_KEY = "jobcontroller_url" CONFIG_SERVING_URL_KEY = "serving_url" CONFIG_SERVING_ENABLE_SSL_KEY = "serving_enable_ssl" CONFIG_SERVING_SERVER_SSL_CERT_KEY = "serving_server_ssl_cert" @@ -76,8 +76,8 @@ class AuthProvider(Enum): CONFIG_ENABLE_AUTH_KEY: "False", # Path to certificate(s) to secure connection to Feast Core CONFIG_CORE_SERVER_SSL_CERT_KEY: "", - # Default Feast Job Coordinator URL - CONFIG_JC_SERVER_KEY: "localhost:6570", + # Default Feast Job Controller URL + CONFIG_JOB_CONTROLLER_SERVER_KEY: "localhost:6570", # Default Feast Serving URL CONFIG_SERVING_URL_KEY: "localhost:6565", # Enable or disable TLS/SSL to Feast Serving diff --git a/sdk/python/feast/contrib/job_coordinator/__init__.py b/sdk/python/feast/contrib/job_controller/__init__.py similarity index 100% rename from sdk/python/feast/contrib/job_coordinator/__init__.py rename to sdk/python/feast/contrib/job_controller/__init__.py diff --git a/sdk/python/feast/contrib/job_coordinator/client.py b/sdk/python/feast/contrib/job_controller/client.py similarity index 80% rename from sdk/python/feast/contrib/job_coordinator/client.py rename to sdk/python/feast/contrib/job_controller/client.py index e03fd2435e1..9a9ffbcc84a 100644 --- a/sdk/python/feast/contrib/job_coordinator/client.py +++ b/sdk/python/feast/contrib/job_controller/client.py @@ -8,15 +8,15 @@ CONFIG_CORE_SERVER_SSL_CERT_KEY, CONFIG_ENABLE_AUTH_KEY, CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, - CONFIG_JC_SERVER_KEY, + CONFIG_JOB_CONTROLLER_SERVER_KEY, ) -from feast.contrib.job_coordinator.job import IngestJob +from feast.contrib.job_controller.job import IngestJob from feast.core.CoreService_pb2 import ( ListIngestionJobsRequest, RestartIngestionJobRequest, StopIngestionJobRequest, ) -from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub +from feast.core.CoreService_pb2_grpc import JobControllerServiceStub from feast.feature_set import FeatureSetRef from feast.grpc import auth as feast_auth from feast.grpc.grpc import create_grpc_channel @@ -24,13 +24,13 @@ class Client: """ - JobCoordinator Client: used internally to manage Ingestion Jobs + JobController Client: used internally to manage Ingestion Jobs """ def __init__(self, options=None, **kwargs): """ - JobCoordinatorClient should be initialized with - jc_url: Feast JobCoordinator address + JobControllerClient should be initialized with + jobcontroller_url: Feast JobController address :param options: Configuration options to initialize client with :param kwargs: options in kwargs style @@ -39,7 +39,7 @@ def __init__(self, options=None, **kwargs): options = dict() self._config = Config(options={**options, **kwargs}) - self._jc_service_stub: Optional[JobCoordinatorServiceStub] = None + self._jobcontroller_service_stub: Optional[JobControllerServiceStub] = None self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None # Configure Auth Metadata Plugin if auth is enabled @@ -47,19 +47,19 @@ def __init__(self, options=None, **kwargs): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) @property - def _jc_service(self): - if not self._jc_service_stub: + def _jobcontroller_service(self): + if not self._jobcontroller_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_JC_SERVER_KEY), + url=self._config.get(CONFIG_JOB_CONTROLLER_SERVER_KEY), enable_ssl=self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY), enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), ssl_server_cert_path=self._config.get(CONFIG_CORE_SERVER_SSL_CERT_KEY), auth_metadata_plugin=self._auth_metadata, timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) - self._jc_service_stub = JobCoordinatorServiceStub(channel) + self._jobcontroller_service_stub = JobControllerServiceStub(channel) - return self._jc_service_stub + return self._jobcontroller_service_stub def list_ingest_jobs( self, @@ -90,9 +90,9 @@ def list_ingest_jobs( ) request = ListIngestionJobsRequest(filter=list_filter) # make list request & unpack response - response = self._jc_service.ListIngestionJobs(request, metadata=self._get_grpc_metadata(),) # type: ignore + response = self._jobcontroller_service.ListIngestionJobs(request, metadata=self._get_grpc_metadata(),) # type: ignore ingest_jobs = [ - IngestJob(proto, self._jc_service, auth_metadata_plugin=self._auth_metadata) for proto in response.jobs # type: ignore + IngestJob(proto, self._jobcontroller_service, auth_metadata_plugin=self._auth_metadata) for proto in response.jobs # type: ignore ] return ingest_jobs @@ -109,7 +109,7 @@ def restart_ingest_job(self, job: IngestJob): """ request = RestartIngestionJobRequest(id=job.id) try: - self._jc_service.RestartIngestionJob( + self._jobcontroller_service.RestartIngestionJob( request, metadata=self._get_grpc_metadata(), ) # type: ignore except grpc.RpcError as e: @@ -127,7 +127,7 @@ def stop_ingest_job(self, job: IngestJob): """ request = StopIngestionJobRequest(id=job.id) try: - self._jc_service.StopIngestionJob( + self._jobcontroller_service.StopIngestionJob( request, metadata=self._get_grpc_metadata(), ) # type: ignore except grpc.RpcError as e: diff --git a/sdk/python/feast/contrib/job_coordinator/job.py b/sdk/python/feast/contrib/job_controller/job.py similarity index 96% rename from sdk/python/feast/contrib/job_coordinator/job.py rename to sdk/python/feast/contrib/job_controller/job.py index 8f2c0738daa..8f2800cba6f 100644 --- a/sdk/python/feast/contrib/job_coordinator/job.py +++ b/sdk/python/feast/contrib/job_controller/job.py @@ -5,7 +5,7 @@ from feast import Source from feast.core.CoreService_pb2 import ListIngestionJobsRequest -from feast.core.CoreService_pb2_grpc import JobCoordinatorServiceStub +from feast.core.CoreService_pb2_grpc import JobControllerServiceStub from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.core.Store_pb2 import Store @@ -21,7 +21,7 @@ class IngestJob: def __init__( self, job_proto: IngestJobProto, - core_stub: JobCoordinatorServiceStub, + core_stub: JobControllerServiceStub, auth_metadata_plugin: grpc.AuthMetadataPlugin = None, ): """ diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 5f86e9cf779..f33fe5c2794 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -28,8 +28,8 @@ from pytz import timezone from feast.client import Client -from feast.contrib.job_coordinator.client import Client as JobCoordinatorClient -from feast.contrib.job_coordinator.job import IngestJob +from feast.contrib.job_controller.client import Client as JCClient +from feast.contrib.job_controller.job import IngestJob from feast.core import CoreService_pb2_grpc as Core from feast.core.CoreService_pb2 import ( GetFeastCoreVersionResponse, @@ -75,7 +75,7 @@ CORE_URL = "core.feast.example.com" SERVING_URL = "serving.example.com" -JC_URL = "jc.feast.example.com" +jobcontroller_URL = "jobcontroller.feast.example.com" _PRIVATE_KEY_RESOURCE_PATH = "data/localhost.key" _CERTIFICATE_CHAIN_RESOURCE_PATH = "data/localhost.pem" _ROOT_CERTIFICATE_RESOURCE_PATH = "data/localhost.crt" @@ -108,8 +108,8 @@ def mock_client(self): return client @pytest.fixture - def mock_jc_client(self): - client = JobCoordinatorClient(jc_url=JC_URL) + def mock_jobcontroller_client(self): + client = JCClient(jobcontroller_url=jobcontroller_URL) return client @pytest.fixture @@ -548,17 +548,17 @@ def test_list_features(self, mocked_client, mocker): and set(feature_dtype_list) == set([ValueType.FLOAT, ValueType.STRING]) ) - def test_list_ingest_jobs(self, mock_jc_client, mocker): + def test_list_ingest_jobs(self, mock_jobcontroller_client, mocker): mocker.patch.object( - mock_jc_client, - "_jc_service_stub", - return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), + mock_jobcontroller_client, + "_jobcontroller_service_stub", + return_value=Core.JobControllerServiceStub(grpc.insecure_channel("")), ) feature_set_ref = FeatureSetRef(project="test", name="driver",) mocker.patch.object( - mock_jc_client._jc_service_stub, + mock_jobcontroller_client._jobcontroller_service_stub, "ListIngestionJobs", return_value=ListIngestionJobsResponse( jobs=[ @@ -580,7 +580,7 @@ def test_list_ingest_jobs(self, mock_jc_client, mocker): ) # list ingestion jobs by target feature set reference - ingest_jobs = mock_jc_client.list_ingest_jobs(feature_set_ref=feature_set_ref) + ingest_jobs = mock_jobcontroller_client.list_ingest_jobs(feature_set_ref=feature_set_ref) assert len(ingest_jobs) >= 1 ingest_job = ingest_jobs[0] @@ -592,11 +592,11 @@ def test_list_ingest_jobs(self, mock_jc_client, mocker): and ingest_job.source.source_type == "Kafka" ) - def test_restart_ingest_job(self, mock_jc_client, mocker): + def test_restart_ingest_job(self, mock_jobcontroller_client, mocker): mocker.patch.object( - mock_jc_client, - "_jc_service_stub", - return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), + mock_jobcontroller_client, + "_jobcontroller_service_stub", + return_value=Core.JobControllerServiceStub(grpc.insecure_channel("")), ) ingest_job = IngestJob( @@ -605,17 +605,17 @@ def test_restart_ingest_job(self, mock_jc_client, mocker): external_id="job#2222", status=IngestionJobStatus.ERROR, ), - core_stub=mock_jc_client._jc_service_stub, + core_stub=mock_jobcontroller_client._jobcontroller_service_stub, ) - mock_jc_client.restart_ingest_job(ingest_job) - assert mock_jc_client._jc_service_stub.RestartIngestionJob.called + mock_jobcontroller_client.restart_ingest_job(ingest_job) + assert mock_jobcontroller_client._jobcontroller_service_stub.RestartIngestionJob.called - def test_stop_ingest_job(self, mock_jc_client, mocker): + def test_stop_ingest_job(self, mock_jobcontroller_client, mocker): mocker.patch.object( - mock_jc_client, - "_jc_service_stub", - return_value=Core.JobCoordinatorServiceStub(grpc.insecure_channel("")), + mock_jobcontroller_client, + "_jobcontroller_service_stub", + return_value=Core.JobControllerServiceStub(grpc.insecure_channel("")), ) ingest_job = IngestJob( @@ -624,11 +624,11 @@ def test_stop_ingest_job(self, mock_jc_client, mocker): external_id="job#2222", status=IngestionJobStatus.RUNNING, ), - core_stub=mock_jc_client._jc_service_stub, + core_stub=mock_jobcontroller_client._jobcontroller_service_stub, ) - mock_jc_client.stop_ingest_job(ingest_job) - assert mock_jc_client._jc_service_stub.StopIngestionJob.called + mock_jobcontroller_client.stop_ingest_job(ingest_job) + assert mock_jobcontroller_client._jobcontroller_service_stub.StopIngestionJob.called @pytest.mark.parametrize( "mocked_client", diff --git a/tests/e2e/bq/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py index ac67beab0cf..d3c56edbf3c 100644 --- a/tests/e2e/bq/bq-batch-retrieval.py +++ b/tests/e2e/bq/bq-batch-retrieval.py @@ -18,7 +18,7 @@ from bq.testutils import assert_stats_equal, clear_unsupported_fields from feast.client import Client -from feast.contrib.job_coordinator.client import Client as JCClient +from feast.contrib.job_controller.client import Client as JCClient from feast.core.CoreService_pb2 import ListStoresRequest from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.entity import Entity @@ -42,8 +42,8 @@ def serving_url(pytestconfig): @pytest.fixture(scope="module") -def jc_url(pytestconfig): - return pytestconfig.getoption("jc_url") +def jobcontroller_url(pytestconfig): + return pytestconfig.getoption("jobcontroller_url") @pytest.fixture(scope="module") @@ -503,8 +503,8 @@ def check(): @pytest.fixture(scope="module", autouse=True) -def infra_teardown(pytestconfig, jc_url): - client = JCClient(jc_url=jc_url) +def infra_teardown(pytestconfig, jobcontroller_url): + client = JCClient(jobcontroller_url=jobcontroller_url) marker = pytestconfig.getoption("-m") yield marker diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ab7e171c254..ea2b809f4fe 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,7 +1,7 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") parser.addoption("--serving_url", action="store", default="localhost:6566") - parser.addoption("--jc_url", action="store", default="localhost:6570") + parser.addoption("--jobcontroller_url", action="store", default="localhost:6570") parser.addoption("--allow_dirty", action="store", default="False") parser.addoption( "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/" diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 72f8c07bdad..3bcf4cac8f6 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -17,7 +17,7 @@ from feast.client import Client from feast.config import Config from feast.constants import CONFIG_AUTH_PROVIDER -from feast.contrib.job_coordinator.client import Client as JCClient +from feast.contrib.job_controller.client import Client as JCClient from feast.core import CoreService_pb2 from feast.core.CoreService_pb2 import ApplyFeatureSetResponse, GetFeatureSetResponse from feast.core.CoreService_pb2_grpc import CoreServiceStub @@ -103,8 +103,8 @@ def serving_url(pytestconfig): @pytest.fixture(scope="module") -def jc_url(pytestconfig): - return pytestconfig.getoption("jc_url") +def jobcontroller_url(pytestconfig): + return pytestconfig.getoption("jobcontroller_url") @pytest.fixture(scope="module") @@ -147,8 +147,8 @@ def client(core_url, serving_url, allow_dirty, enable_auth): @pytest.fixture(scope="module") -def jc_client(jc_url): - client = JCClient(jc_url=jc_url) +def jobcontroller_client(jobcontroller_url): + client = JCClient(jobcontroller_url=jobcontroller_url) return client @@ -965,12 +965,12 @@ def try_get_features(): @pytest.mark.timeout(300) @pytest.mark.run(order=35) -def test_all_types_ingest_jobs(jc_client, client, all_types_dataframe): +def test_all_types_ingest_jobs(jobcontroller_client, client, all_types_dataframe): # list ingestion jobs given featureset client.set_project(PROJECT_NAME) all_types_fs = client.get_feature_set(name="all_types") - ingest_jobs = jc_client.list_ingest_jobs( + ingest_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs) ) # filter ingestion jobs to only those that are running @@ -983,14 +983,14 @@ def test_all_types_ingest_jobs(jc_client, client, all_types_dataframe): # restart ingestion ingest_job # restart means stop current job # (replacement will be automatically spawned) - jc_client.restart_ingest_job(ingest_job) + jobcontroller_client.restart_ingest_job(ingest_job) # wait for replacement to be created time.sleep(15) # should be more than polling_interval # id without timestamp part # that remains the same between jobs shared_id = "-".join(ingest_job.id.split("-")[:-1]) - ingest_jobs = jc_client.list_ingest_jobs( + ingest_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs) ) replacement_jobs = [ @@ -1008,7 +1008,7 @@ def test_all_types_ingest_jobs(jc_client, client, all_types_dataframe): assert replacement_job.status == IngestionJobStatus.RUNNING # stop ingestion ingest_job - jc_client.stop_ingest_job(replacement_job) + jobcontroller_client.stop_ingest_job(replacement_job) replacement_job.wait(IngestionJobStatus.ABORTED) assert replacement_job.status == IngestionJobStatus.ABORTED @@ -1267,7 +1267,7 @@ def test_list_entities_and_features(client): @pytest.mark.timeout(500) @pytest.mark.run(order=70) -def test_sources_deduplicate_ingest_jobs(client, jc_client, kafka_brokers): +def test_sources_deduplicate_ingest_jobs(client, jobcontroller_client, kafka_brokers): shared_source = KafkaSource(kafka_brokers, "dup_shared") dup_source_fs_1 = FeatureSet( name="duplicate_source_fs_1", @@ -1279,12 +1279,12 @@ def test_sources_deduplicate_ingest_jobs(client, jc_client, kafka_brokers): dup_source_fs_2.name = "duplicate_source_fs_2" def is_same_jobs(): - fs_1_jobs = jc_client.list_ingest_jobs( + fs_1_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_1.name, project=dup_source_fs_1.project ) ) - fs_2_jobs = jc_client.list_ingest_jobs( + fs_2_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_2.name, project=dup_source_fs_2.project ) @@ -1304,12 +1304,12 @@ def is_same_jobs(): return same def is_different_jobs(): - fs_1_jobs = jc_client.list_ingest_jobs( + fs_1_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_1.name, project=dup_source_fs_1.project ) ) - fs_2_jobs = jc_client.list_ingest_jobs( + fs_2_jobs = jobcontroller_client.list_ingest_jobs( feature_set_ref=FeatureSetRef( name=dup_source_fs_2.name, project=dup_source_fs_2.project ) From 38e7df8f872d94e944e336e4dc7a432d886c6dde Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 20 Aug 2020 15:57:34 +0800 Subject: [PATCH 21/26] format --- infra/docker-compose/.env.sample | 2 +- infra/docker-compose/docker-compose.yml | 2 +- .../runner/option/RunnerConfig.java | 4 +-- .../service/JobControllerServiceTest.java | 34 +++++++++++-------- sdk/python/docs/conf.py | 12 +++---- sdk/python/tests/test_client.py | 12 +++++-- 6 files changed, 38 insertions(+), 28 deletions(-) diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index f2d02ceadee..4dcca0b60e6 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,7 +2,7 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=0.6.2 GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json FEAST_CORE_CONFIG=./core/core.yml -FEAST_jobcontroller_CONFIG=./jobcontroller/jobcontroller.yml +FEAST_JOB_CONTROLLER_CONFIG=./jobcontroller/jobcontroller.yml FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml FEAST_HISTORICAL_SERVING_ENABLED=false FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 0efb9eb3296..197ee56d175 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -24,7 +24,7 @@ services: jobcontroller: image: gcr.io/kf-feast/feast-jobcontroller:${FEAST_VERSION} volumes: - - ${FEAST_jobcontroller_CONFIG}:/etc/feast/application.yml + - ${FEAST_JOB_CONTROLLER_CONFIG}:/etc/feast/application.yml - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json environment: GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java index dba49a5cd8d..336f133efce 100644 --- a/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java +++ b/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java @@ -24,8 +24,8 @@ /** * Value class containing the application-default configuration for a runner. When a job is started - * by jobcontroller, all fields in the object will be converted into --key=value args to seed the beam pipeline - * options. + * by jobcontroller, all fields in the object will be converted into --key=value args to seed the + * beam pipeline options. */ public abstract class RunnerConfig { diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java index 8abdb62a2db..a99cb954b8e 100644 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java @@ -67,8 +67,8 @@ public class JobControllerServiceTest { @Mock CoreServiceGrpc.CoreServiceBlockingStub specService; private FeastProperties feastProperties; - private JobControllerService jobcontrollersWithConsolidation; - private JobControllerService jobcontrollersWithJobPerStore; + private JobControllerService controllerWithConsolidation; + private JobControllerService controllerWithJobPerStore; @BeforeEach public void setUp() { @@ -98,7 +98,7 @@ public void setUp() { when(jobManager.listRunningJobs()).thenReturn(Collections.emptyList()); jobRepository = new InMemoryJobRepository(jobManager); - jobcontrollersWithConsolidation = + controllerWithConsolidation = new JobControllerService( jobRepository, specService, @@ -107,7 +107,7 @@ public void setUp() { new ConsolidatedJobStrategy(jobRepository), mock(KafkaTemplate.class)); - jobcontrollersWithJobPerStore = + controllerWithJobPerStore = new JobControllerService( jobRepository, specService, @@ -122,7 +122,8 @@ public void shouldDoNothingIfNoStoresFound() { when(specService.listStores(any())).thenReturn(ListStoresResponse.newBuilder().build()); List jobTasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks(jobcontrollersWithConsolidation.getSourceToStoreMappings()); + controllerWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.getSourceToStoreMappings()); assertThat(jobTasks, hasSize(0)); } @@ -140,7 +141,8 @@ public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocol .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List jobTasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks(jobcontrollersWithConsolidation.getSourceToStoreMappings()); + controllerWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.getSourceToStoreMappings()); assertThat(jobTasks, hasSize(0)); } @@ -171,7 +173,7 @@ public void shouldGroupJobsBySource() { .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); ArrayList>> pairs = - Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, hasSize(2)); assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store)))); @@ -219,7 +221,7 @@ public void shouldUseStoreSubscriptionToMapStore() { .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); ArrayList>> pairs = - Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, hasSize(2)); assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store1)))); @@ -243,7 +245,7 @@ public void shouldCheckStatusOfAbortingJob() { jobRepository.add(job); List tasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CheckStatus is expected", tasks.get(0) instanceof UpdateJobStatusTask); @@ -262,7 +264,7 @@ public void shouldUpgradeJobWhenNeeded() { jobRepository.add(job); List tasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); @@ -281,7 +283,7 @@ public void shouldCreateJobIfNoRunning() { .thenReturn(ListFeatureSetsResponse.newBuilder().build()); List tasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); @@ -312,7 +314,8 @@ public void shouldCreateJobPerStore() throws InvalidProtocolBufferException { .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); List jobTasks = - jobcontrollersWithJobPerStore.makeJobUpdateTasks(jobcontrollersWithJobPerStore.getSourceToStoreMappings()); + controllerWithJobPerStore.makeJobUpdateTasks( + controllerWithJobPerStore.getSourceToStoreMappings()); int hash = Objects.hash( @@ -358,7 +361,7 @@ public void shouldCloneRunningJobOnUpgrade() throws InvalidProtocolBufferExcepti jobRepository.add(existingJob); List jobTasks = - jobcontrollersWithConsolidation.makeJobUpdateTasks( + controllerWithConsolidation.makeJobUpdateTasks( ImmutableList.of(Pair.of(source, ImmutableSet.of(store1, store2)))); assertThat(jobTasks, hasSize(1)); @@ -384,7 +387,8 @@ public void shouldSelectOnlyFeatureSetsThatJobManagerSubscribedTo() { .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2, featureSet3)) .build()); - List featureSetsForStore = jobcontrollersWithConsolidation.getFeatureSetsForStore(store); + List featureSetsForStore = + controllerWithConsolidation.getFeatureSetsForStore(store); assertThat(featureSetsForStore, containsInAnyOrder(featureSet1, featureSet2)); } @@ -426,7 +430,7 @@ public void shouldSelectOnlyStoresThatWhitelisted() { .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet2).build()); ArrayList>> pairs = - Lists.newArrayList(jobcontrollersWithConsolidation.getSourceToStoreMappings()); + Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); assertThat(pairs, containsInAnyOrder(Pair.of(source1, ImmutableSet.of(store1)))); } diff --git a/sdk/python/docs/conf.py b/sdk/python/docs/conf.py index 8a72b7456b5..c4b9f8fde28 100644 --- a/sdk/python/docs/conf.py +++ b/sdk/python/docs/conf.py @@ -61,9 +61,9 @@ master_doc = "index" # General information about the project. -project = u"Feast" -copyright = u"2019, Feast Authors" -author = u"Feast Authors" +project = "Feast" +copyright = "2019, Feast Authors" +author = "Feast Authors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -145,7 +145,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, "Feast.tex", u"Feast Documentation", u"Feast Authors", "manual") + (master_doc, "Feast.tex", "Feast Documentation", "Feast Authors", "manual") ] @@ -153,7 +153,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "feast", u"Feast Documentation", [author], 1)] +man_pages = [(master_doc, "feast", "Feast Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -165,7 +165,7 @@ ( master_doc, "Feast", - u"Feast Documentation", + "Feast Documentation", author, "Feast", "One line description of project.", diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index f33fe5c2794..23603c9765a 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -580,7 +580,9 @@ def test_list_ingest_jobs(self, mock_jobcontroller_client, mocker): ) # list ingestion jobs by target feature set reference - ingest_jobs = mock_jobcontroller_client.list_ingest_jobs(feature_set_ref=feature_set_ref) + ingest_jobs = mock_jobcontroller_client.list_ingest_jobs( + feature_set_ref=feature_set_ref + ) assert len(ingest_jobs) >= 1 ingest_job = ingest_jobs[0] @@ -609,7 +611,9 @@ def test_restart_ingest_job(self, mock_jobcontroller_client, mocker): ) mock_jobcontroller_client.restart_ingest_job(ingest_job) - assert mock_jobcontroller_client._jobcontroller_service_stub.RestartIngestionJob.called + assert ( + mock_jobcontroller_client._jobcontroller_service_stub.RestartIngestionJob.called + ) def test_stop_ingest_job(self, mock_jobcontroller_client, mocker): mocker.patch.object( @@ -628,7 +632,9 @@ def test_stop_ingest_job(self, mock_jobcontroller_client, mocker): ) mock_jobcontroller_client.stop_ingest_job(ingest_job) - assert mock_jobcontroller_client._jobcontroller_service_stub.StopIngestionJob.called + assert ( + mock_jobcontroller_client._jobcontroller_service_stub.StopIngestionJob.called + ) @pytest.mark.parametrize( "mocked_client", From 1575ee2f20ebec49370544717ed302db1124cddc Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 20 Aug 2020 17:11:51 +0800 Subject: [PATCH 22/26] jobcontroller docker compose config --- .../jobcontroller/{jc.yml => jobcontroller.yml} | 0 infra/scripts/test-load.sh | 4 ++-- job-controller/src/main/resources/banner.txt | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) rename infra/docker-compose/jobcontroller/{jc.yml => jobcontroller.yml} (100%) diff --git a/infra/docker-compose/jobcontroller/jc.yml b/infra/docker-compose/jobcontroller/jobcontroller.yml similarity index 100% rename from infra/docker-compose/jobcontroller/jc.yml rename to infra/docker-compose/jobcontroller/jobcontroller.yml diff --git a/infra/scripts/test-load.sh b/infra/scripts/test-load.sh index e8dbebb2c24..51a24691067 100755 --- a/infra/scripts/test-load.sh +++ b/infra/scripts/test-load.sh @@ -67,10 +67,10 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett "${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120 # Get Feast Job Controller container IP address -export FEAST_jobcontroller_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1) +export FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1) # Wait for Feast Job Controller to be ready -"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_jobcontroller_CONTAINER_IP_ADDRESS}:6570 --timeout=120 +"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS}:6570 --timeout=120 # Get Feast Online Serving container IP address export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1) diff --git a/job-controller/src/main/resources/banner.txt b/job-controller/src/main/resources/banner.txt index c2bc81e3e0d..3ae34145853 100644 --- a/job-controller/src/main/resources/banner.txt +++ b/job-controller/src/main/resources/banner.txt @@ -13,9 +13,9 @@ ╚█████╔╝╚██████╔╝██████╔╝ ╚════╝ ╚═════╝ ╚═════╝ - ██████╗ ██████╗ ██████╗ ██████╗ ██████╗ ██╗███╗ ██╗ █████╗ ████████╗ ██████╗ ██████╗ -██╔════╝██╔═══██╗██╔═══██╗██╔══██╗██╔══██╗██║████╗ ██║██╔══██╗╚══██╔══╝██╔═══██╗██╔══██╗ -██║ ██║ ██║██║ ██║██████╔╝██║ ██║██║██╔██╗ ██║███████║ ██║ ██║ ██║██████╔╝ -██║ ██║ ██║██║ ██║██╔══██╗██║ ██║██║██║╚██╗██║██╔══██║ ██║ ██║ ██║██╔══██╗ -╚██████╗╚██████╔╝╚██████╔╝██║ ██║██████╔╝██║██║ ╚████║██║ ██║ ██║ ╚██████╔╝██║ ██║ - ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚═════╝ ╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ + ██████╗ ██████╗ ███╗ ██╗████████╗██████╗ ██████╗ ██╗ ██╗ ███████╗██████╗ +██╔════╝██╔═══██╗████╗ ██║╚══██╔══╝██╔══██╗██╔═══██╗██║ ██║ ██╔════╝██╔══██╗ +██║ ██║ ██║██╔██╗ ██║ ██║ ██████╔╝██║ ██║██║ ██║ █████╗ ██████╔╝ +██║ ██║ ██║██║╚██╗██║ ██║ ██╔══██╗██║ ██║██║ ██║ ██╔══╝ ██╔══██╗ +╚██████╗╚██████╔╝██║ ╚████║ ██║ ██║ ██║╚██████╔╝███████╗███████╗███████╗██║ ██║ + ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝ From 1b2a615048ba59c5bbfa74cce64f05c96dfbaf93 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 20 Aug 2020 17:54:10 +0800 Subject: [PATCH 23/26] pr comments --- .../common/it/{SimpleJCClient.java => SimpleJcClient.java} | 4 ++-- .../main/java/feast/common/models/FeatureSetReference.java | 7 +++++++ .../java/feast/jobcontroller/service/JobControllerIT.java | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) rename common-test/src/main/java/feast/common/it/{SimpleJCClient.java => SimpleJcClient.java} (93%) diff --git a/common-test/src/main/java/feast/common/it/SimpleJCClient.java b/common-test/src/main/java/feast/common/it/SimpleJcClient.java similarity index 93% rename from common-test/src/main/java/feast/common/it/SimpleJCClient.java rename to common-test/src/main/java/feast/common/it/SimpleJcClient.java index 31c62934f7f..247e3b01b6b 100644 --- a/common-test/src/main/java/feast/common/it/SimpleJCClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleJcClient.java @@ -21,10 +21,10 @@ import feast.proto.core.JobControllerServiceGrpc; import java.util.List; -public class SimpleJCClient { +public class SimpleJcClient { private final JobControllerServiceGrpc.JobControllerServiceBlockingStub stub; - public SimpleJCClient(JobControllerServiceGrpc.JobControllerServiceBlockingStub stub) { + public SimpleJcClient(JobControllerServiceGrpc.JobControllerServiceBlockingStub stub) { this.stub = stub; } diff --git a/common/src/main/java/feast/common/models/FeatureSetReference.java b/common/src/main/java/feast/common/models/FeatureSetReference.java index 42722d58dc8..ea01bf5cf73 100644 --- a/common/src/main/java/feast/common/models/FeatureSetReference.java +++ b/common/src/main/java/feast/common/models/FeatureSetReference.java @@ -48,6 +48,13 @@ public static FeatureSetReference of(String projectName, String featureSetName) return FeatureSetReference.of(projectName, featureSetName, -1); } + /** + * Parse string representation of FeatureSetReference that expected to have format + * <ProjectName>/<FeatureSetName>. If project's not given - default will be used. + * + * @param reference string representation + * @return construct {@link FeatureSetReference} + */ public static FeatureSetReference parse(String reference) { String[] split = reference.split("/", 2); if (split.length == 1) { diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java index ead71d1f67a..111b5783479 100644 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java @@ -84,7 +84,7 @@ public class JobControllerIT extends BaseIT { static CoreServiceGrpc.CoreServiceBlockingStub stub; static List specsMailbox = new ArrayList<>(); static SimpleCoreClient coreApiClient; - static SimpleJCClient apiClient; + static SimpleJcClient apiClient; static int corePort = SocketUtils.findAvailableTcpPort(); static ExternalApp coreApp = @@ -110,7 +110,7 @@ public static void globalSetUp(@Value("${grpc.server.port}") int localPort) { coreApiClient = new SimpleCoreClient(stub); apiClient = - new SimpleJCClient( + new SimpleJcClient( JobControllerServiceGrpc.newBlockingStub( ManagedChannelBuilder.forAddress("localhost", localPort).usePlaintext().build())); } From bcc4625251fbbfd45abcdc3f37bc90b767e9326c Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 21 Aug 2020 09:31:27 +0800 Subject: [PATCH 24/26] pr comments --- ...viceImpl.java => JobControllerServiceImpl.java} | 4 ++-- .../service/JobControllerService.java | 14 ++++++++++---- .../jobcontroller/service/JobControllerIT.java | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) rename job-controller/src/main/java/feast/jobcontroller/grpc/{CoreServiceImpl.java => JobControllerServiceImpl.java} (97%) diff --git a/job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java b/job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java similarity index 97% rename from job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java rename to job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java index 607e6fbe7f7..8d4a6fc588f 100644 --- a/job-controller/src/main/java/feast/jobcontroller/grpc/CoreServiceImpl.java +++ b/job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java @@ -31,12 +31,12 @@ /** Implementation of the feast core GRPC service. */ @Slf4j @GrpcService(interceptors = {GrpcMessageInterceptor.class}) -public class CoreServiceImpl extends JobControllerServiceImplBase { +public class JobControllerServiceImpl extends JobControllerServiceImplBase { private JobService jobService; @Autowired - public CoreServiceImpl(JobService jobService) { + public JobControllerServiceImpl(JobService jobService) { this.jobService = jobService; } diff --git a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java index 7cd896b3581..99d689dcf66 100644 --- a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java +++ b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java @@ -94,11 +94,17 @@ public JobControllerService( this.whitelistedStores = feastProperties.getJobs().getController().getWhitelistedStores(); this.currentVersion = feastProperties.getVersion(); this.jobLabels = new HashMap<>(feastProperties.getJobs().getController().getJobSelector()); - this.jobLabels.put(VERSION_LABEL, getCurrentFeastVersion()); + this.jobLabels.put(VERSION_LABEL, getVersionLabel()); } - private String getCurrentFeastVersion() { - return this.currentVersion.replace(".", "-"); + /** + * Generate version label value that must conform to regular expression + * [\\p{Ll}\\p{Lo}\\p{N}_-]{0,63} from current version of Feast. + * + * @return lower-cased version with replaced dots + */ + private String getVersionLabel() { + return this.currentVersion.replace(".", "-").toLowerCase(); } /** @@ -239,7 +245,7 @@ private boolean jobRequiresUpgrade(Job job, Set stores) { return true; } - if (!getCurrentFeastVersion().equals(job.getLabels().get(VERSION_LABEL))) { + if (!getVersionLabel().equals(job.getLabels().get(VERSION_LABEL))) { return true; } diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java index 111b5783479..cad059f8c02 100644 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java +++ b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java @@ -97,7 +97,7 @@ public class JobControllerIT extends BaseIT { @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) { - registry.add("feast.corePort", () -> corePort); + registry.add("feast.core-port", () -> corePort); } @BeforeAll From 48c31d359b213b064eb1780855e639fb57d3b6c7 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 21 Aug 2020 09:34:17 +0800 Subject: [PATCH 25/26] label regex --- .../java/feast/jobcontroller/service/JobControllerService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java index 99d689dcf66..fff9e760f87 100644 --- a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java +++ b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java @@ -104,7 +104,7 @@ public JobControllerService( * @return lower-cased version with replaced dots */ private String getVersionLabel() { - return this.currentVersion.replace(".", "-").toLowerCase(); + return this.currentVersion.replace(".", "-").toLowerCase().substring(0, 63); } /** From 028f9d282e100717d0cb7b75d47dceb4ee1b12d3 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 21 Aug 2020 10:05:56 +0800 Subject: [PATCH 26/26] fix version label --- .../feast/jobcontroller/service/JobControllerService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java index fff9e760f87..d2e7dbd0394 100644 --- a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java +++ b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java @@ -104,7 +104,10 @@ public JobControllerService( * @return lower-cased version with replaced dots */ private String getVersionLabel() { - return this.currentVersion.replace(".", "-").toLowerCase().substring(0, 63); + return this.currentVersion + .replace(".", "-") + .toLowerCase() + .substring(0, Math.min(this.currentVersion.length(), 63)); } /**