2323import feast .ingestion .transform .fn .KafkaRecordToFeatureRowDoFn ;
2424import feast .storage .api .writer .FailedElement ;
2525import feast .types .FeatureRowProto .FeatureRow ;
26+ import java .util .Optional ;
2627import org .apache .beam .sdk .io .kafka .KafkaIO ;
2728import org .apache .beam .sdk .transforms .PTransform ;
2829import org .apache .beam .sdk .transforms .ParDo ;
@@ -41,6 +42,8 @@ public abstract class ReadFromSource extends PTransform<PBegin, PCollectionTuple
4142
4243 public abstract TupleTag <FailedElement > getFailureTag ();
4344
45+ public abstract Optional <String > getConsumerGroupId ();
46+
4447 public static Builder newBuilder () {
4548 return new AutoValue_ReadFromSource .Builder ();
4649 }
@@ -54,6 +57,8 @@ public abstract static class Builder {
5457
5558 public abstract Builder setFailureTag (TupleTag <FailedElement > failureTag );
5659
60+ public abstract Builder setConsumerGroupId (String consumerGroupId );
61+
5762 abstract ReadFromSource autobuild ();
5863
5964 public ReadFromSource build () {
@@ -83,7 +88,10 @@ public PCollectionTuple expand(PBegin input) {
8388 .withConsumerConfigUpdates (
8489 ImmutableMap .of (
8590 "group.id" ,
86- generateConsumerGroupId (input .getPipeline ().getOptions ().getJobName ())))
91+ getConsumerGroupId ().isPresent ()
92+ ? getConsumerGroupId ().get ()
93+ : generateConsumerGroupId (
94+ input .getPipeline ().getOptions ().getJobName ())))
8795 .withReadCommitted ()
8896 .commitOffsetsInFinalize ())
8997 .apply (
@@ -99,4 +107,11 @@ public PCollectionTuple expand(PBegin input) {
99107 private String generateConsumerGroupId (String jobName ) {
100108 return "feast_import_job_" + jobName ;
101109 }
110+
111+ /** Get a stable Kafka consumer group ID, so restarting jobs can resume offsets. */
112+ public static String generateConsumerGroupId (SourceType sourceType , String storeName ) {
113+ return String .format ("feast_import_job_%s_%s" , sourceType , storeName )
114+ .replaceAll (" " , "_" )
115+ .toLowerCase ();
116+ }
102117}
0 commit comments