Skip to content

Commit 8ef3e2a

Browse files
author
zhilingc
committed
Re-implement metrics for ingestion, remove more redundant code
1 parent 225582f commit 8ef3e2a

16 files changed

Lines changed: 99 additions & 2019 deletions

‎ingestion/src/main/java/feast/ingestion/ImportJob.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import feast.ingestion.transform.ReadFromSource;
88
import feast.ingestion.transform.WriteFailedElementToBigQuery;
99
import feast.ingestion.transform.WriteToStore;
10+
import feast.ingestion.transform.metrics.WriteMetricsTransform;
1011
import feast.ingestion.utils.ResourceUtil;
1112
import feast.ingestion.utils.SpecUtil;
1213
import feast.ingestion.utils.StoreUtil;
@@ -47,6 +48,7 @@ public static PipelineResult runPipeline(ImportOptions options)
4748
* 1. Read messages from Feast Source as FeatureRow
4849
* 2. Write FeatureRow to the corresponding Store
4950
* 3. Write elements that failed to be processed to a dead letter queue.
51+
* 4. Write metrics to a metrics sink
5052
*/
5153

5254
PipelineOptionsValidator.validate(ImportOptions.class, options);
@@ -95,6 +97,15 @@ public static PipelineResult runPipeline(ImportOptions options)
9597
.setTableSpec(options.getDeadLetterTableSpec())
9698
.build());
9799
}
100+
101+
// Step 4. Write metrics to a metrics sink.
102+
convertedFeatureRows
103+
.apply("WriteMetrics", WriteMetricsTransform.newBuilder()
104+
.setFeatureSetSpec(featureSet)
105+
.setStoreName(store.getName())
106+
.setSuccessTag(FEATURE_ROW_OUT)
107+
.setFailureTag(DEADLETTER_OUT)
108+
.build());
98109
}
99110
}
100111

‎ingestion/src/main/java/feast/ingestion/transform/FilterFeatureRow.java‎

Lines changed: 0 additions & 52 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/ReadFeatureRow.java‎

Lines changed: 0 additions & 132 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/ToFeatureRowExtended.java‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/WriteFeaturesTransform.java‎

Lines changed: 0 additions & 85 deletions
This file was deleted.

0 commit comments

Comments
 (0)