Skip to content

Add configurable prefix to Consumer Group in IngestionJob's Kafka reader #969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 3 commits into from
Sep 2, 2020
Merged

Add configurable prefix to Consumer Group in IngestionJob's Kafka reader #969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4105a31
Add JobId prefix setting
terryyylim Sep 1, 2020
cde8bbe
Update job controller tests
terryyylim Sep 1, 2020
998d23c
Address PR comments
terryyylim Sep 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ public feast.jobcontroller.runner.Runner getType() {
/* Population job metric properties */
private MetricsProperties metrics;

@NotNull
/* Prefix for JobId to separate consumer groups for independent jobs running in parallel */
private String jobIdPrefix;

/* Timeout in seconds for each attempt to update or submit a new job to the runner */
@Positive private long jobUpdateTimeoutSeconds;

Expand Down
5 changes: 3 additions & 2 deletions job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,11 @@ public JobGroupingStrategy getJobGroupingStrategy(
FeastProperties feastProperties, JobRepository jobRepository) {
Boolean shouldConsolidateJobs =
feastProperties.getJobs().getController().getConsolidateJobsPerSource();
FeastProperties.JobProperties jobProperties = feastProperties.getJobs();
if (shouldConsolidateJobs) {
return new ConsolidatedJobStrategy(jobRepository);
return new ConsolidatedJobStrategy(jobRepository, jobProperties);
} else {
return new JobPerStoreStrategy(jobRepository);
return new JobPerStoreStrategy(jobRepository, jobProperties);
}
}

Expand Down
9 changes: 8 additions & 1 deletion job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package feast.jobcontroller.runner;

import feast.jobcontroller.config.FeastProperties.JobProperties;
import feast.jobcontroller.dao.JobRepository;
import feast.jobcontroller.model.Job;
import feast.jobcontroller.model.JobStatus;
Expand All @@ -38,9 +39,11 @@
*/
public class ConsolidatedJobStrategy implements JobGroupingStrategy {
private final JobRepository jobRepository;
private final JobProperties jobProperties;

public ConsolidatedJobStrategy(JobRepository jobRepository) {
public ConsolidatedJobStrategy(JobRepository jobRepository, JobProperties jobProperties) {
this.jobRepository = jobRepository;
this.jobProperties = jobProperties;
}

@Override
Expand Down Expand Up @@ -71,6 +74,10 @@ private String createJobId(SourceProto.Source source) {
source.getKafkaSourceConfig().getBootstrapServers(),
source.getKafkaSourceConfig().getTopic()),
dateSuffix);
if (this.jobProperties.getJobIdPrefix() != null
&& !this.jobProperties.getJobIdPrefix().isEmpty()) {
jobId = this.jobProperties.getJobIdPrefix() + "-" + jobId;
}
return jobId.replaceAll("_store", "-").toLowerCase();
}

Expand Down
9 changes: 8 additions & 1 deletion job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package feast.jobcontroller.runner;

import com.google.common.collect.Lists;
import feast.jobcontroller.config.FeastProperties.JobProperties;
import feast.jobcontroller.dao.JobRepository;
import feast.jobcontroller.model.Job;
import feast.jobcontroller.model.JobStatus;
Expand All @@ -38,9 +39,11 @@
*/
public class JobPerStoreStrategy implements JobGroupingStrategy {
private final JobRepository jobRepository;
private final JobProperties jobProperties;

public JobPerStoreStrategy(JobRepository jobRepository) {
public JobPerStoreStrategy(JobRepository jobRepository, JobProperties jobProperties) {
this.jobRepository = jobRepository;
this.jobProperties = jobProperties;
}

@Override
Expand Down Expand Up @@ -78,6 +81,10 @@ private String createJobId(SourceProto.Source source, Iterable<StoreProto.Store>
source.getKafkaSourceConfig().getTopic()),
Lists.newArrayList(stores).get(0).getName(),
dateSuffix);
if (this.jobProperties.getJobIdPrefix() != null
&& !this.jobProperties.getJobIdPrefix().isEmpty()) {
jobId = this.jobProperties.getJobIdPrefix() + "-" + jobId;
}
return jobId.replaceAll("_store", "-").toLowerCase();
}

Expand Down
3 changes: 3 additions & 0 deletions job-controller/src/main/resources/application.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ feast:
# Enabling JobManagement
enabled: true

# Configurable Prefix for JobId to allow jobs working in parallel to separate their Kafka consumer groups
job_id_prefix: ""

# Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner.
polling_interval_milliseconds: 60000

Expand Down
4 changes: 4 additions & 0 deletions job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
@SpringBootTest(
properties = {
"feast.jobs.enabled=true",
"feast.jobs.job_id_prefix=test-prefix",
"feast.jobs.polling_interval_milliseconds=1000",
"feast.stream.specsOptions.notifyIntervalMilliseconds=1000",
"feast.jobs.controller.consolidate-jobs-per-source=true",
Expand Down Expand Up @@ -166,6 +167,9 @@ public void shouldCreateJobForNewSource() {

assertThat(
jobManager.getAllJobs(), containsInAnyOrder(hasProperty("id", equalTo(job.getId()))));
assertThat(
jobManager.getAllJobs(),
containsInAnyOrder(hasProperty("id", containsString("test-prefix"))));
}

@Test
Expand Down
6 changes: 4 additions & 2 deletions job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public void setUp() {
feastProperties = new FeastProperties();
JobProperties jobProperties = new JobProperties();
jobProperties.setJobUpdateTimeoutSeconds(5);
jobProperties.setJobIdPrefix("test-prefix");

FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector selector =
new FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector();
Expand Down Expand Up @@ -104,7 +105,7 @@ public void setUp() {
specService,
jobManager,
feastProperties,
new ConsolidatedJobStrategy(jobRepository),
new ConsolidatedJobStrategy(jobRepository, feastProperties.getJobs()),
mock(KafkaTemplate.class));

controllerWithJobPerStore =
Expand All @@ -113,7 +114,7 @@ public void setUp() {
specService,
jobManager,
feastProperties,
new JobPerStoreStrategy(jobRepository),
new JobPerStoreStrategy(jobRepository, feastProperties.getJobs()),
mock(KafkaTemplate.class));
}

Expand Down Expand Up @@ -286,6 +287,7 @@ public void shouldCreateJobIfNoRunning() {
controllerWithConsolidation.makeJobUpdateTasks(
ImmutableList.of(Pair.of(source, ImmutableSet.of(store))));

assertThat(tasks.get(0).getJob().getId(), containsString("test-prefix"));
assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask);
}

Expand Down