Skip to content

Use stable Kafka consumer group ID for job #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kishanpradhan wants to merge 1 commit into from
Closed

Use stable Kafka consumer group ID for job #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ingestion/src/main/java/feast/ingestion/ImportJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti
"ReadFeatureRowFromSource",
ReadFromSource.newBuilder()
.setSource(source)
.setConsumerGroupId(
ReadFromSource.generateConsumerGroupId(source.getType(), store.getName()))
.setSuccessTag(FEATURE_ROW_OUT)
.setFailureTag(DEADLETTER_OUT)
.build());
Expand Down
17 changes: 16 additions & 1 deletion ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import feast.ingestion.transform.fn.KafkaRecordToFeatureRowDoFn;
import feast.storage.api.writer.FailedElement;
import feast.types.FeatureRowProto.FeatureRow;
import java.util.Optional;
import org.apache.beam.sdk.io.kafka.KafkaIO;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
Expand All @@ -41,6 +42,8 @@ public abstract class ReadFromSource extends PTransform<PBegin, PCollectionTuple

public abstract TupleTag<FailedElement> getFailureTag();

public abstract Optional<String> getConsumerGroupId();

public static Builder newBuilder() {
return new AutoValue_ReadFromSource.Builder();
}
Expand All @@ -54,6 +57,8 @@ public abstract static class Builder {

public abstract Builder setFailureTag(TupleTag<FailedElement> failureTag);

public abstract Builder setConsumerGroupId(String consumerGroupId);

abstract ReadFromSource autobuild();

public ReadFromSource build() {
Expand Down Expand Up @@ -83,7 +88,10 @@ public PCollectionTuple expand(PBegin input) {
.withConsumerConfigUpdates(
ImmutableMap.of(
"group.id",
generateConsumerGroupId(input.getPipeline().getOptions().getJobName())))
getConsumerGroupId().isPresent()
? getConsumerGroupId().get()
: generateConsumerGroupId(
input.getPipeline().getOptions().getJobName())))
.withReadCommitted()
.commitOffsetsInFinalize())
.apply(
Expand All @@ -99,4 +107,11 @@ public PCollectionTuple expand(PBegin input) {
private String generateConsumerGroupId(String jobName) {
return "feast_import_job_" + jobName;
}

/** Get a stable Kafka consumer group ID, so restarting jobs can resume offsets. */
public static String generateConsumerGroupId(SourceType sourceType, String storeName) {
return String.format("feast_import_job_%s_%s", sourceType, storeName)
.replaceAll(" ", "_")
.toLowerCase();
}
}