Skip to content

Commit 11114c5

Browse files
timsfeast-ci-bot
authored andcommitted
allow empty string to select a NoOp write transform (#30)
* allow empty string to select a NoOp write transform * reformat to google styles
1 parent 7d4eb26 commit 11114c5

7 files changed

Lines changed: 361 additions & 66 deletions

File tree

ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@
1919

2020
import com.google.common.base.Preconditions;
2121
import com.google.common.collect.Lists;
22+
import feast.ingestion.exceptions.ErrorsHandler;
23+
import feast.ingestion.model.Specs;
24+
import feast.ingestion.transform.FeatureIO.Write;
25+
import feast.ingestion.transform.SplitFeatures.MultiOutputSplit;
26+
import feast.ingestion.values.PFeatureRows;
27+
import feast.specs.FeatureSpecProto.FeatureSpec;
28+
import feast.specs.StorageSpecProto.StorageSpec;
29+
import feast.storage.FeatureStore;
30+
import feast.storage.noop.NoOpIO;
31+
import feast.types.FeatureRowExtendedProto.Attempt;
32+
import feast.types.FeatureRowExtendedProto.Error;
33+
import feast.types.FeatureRowExtendedProto.FeatureRowExtended;
2234
import java.util.Collection;
2335
import java.util.HashMap;
2436
import java.util.List;
@@ -35,29 +47,19 @@
3547
import org.apache.beam.sdk.values.PCollectionList;
3648
import org.apache.beam.sdk.values.PCollectionTuple;
3749
import org.apache.beam.sdk.values.TupleTag;
38-
import feast.ingestion.exceptions.ErrorsHandler;
39-
import feast.ingestion.model.Specs;
40-
import feast.ingestion.transform.FeatureIO.Write;
41-
import feast.ingestion.transform.SplitFeatures.MultiOutputSplit;
42-
import feast.ingestion.values.PFeatureRows;
43-
import feast.specs.FeatureSpecProto.FeatureSpec;
44-
import feast.specs.StorageSpecProto.StorageSpec;
45-
import feast.storage.FeatureStore;
46-
import feast.storage.noop.NoOpIO;
47-
import feast.types.FeatureRowExtendedProto.Attempt;
48-
import feast.types.FeatureRowExtendedProto.Error;
49-
import feast.types.FeatureRowExtendedProto.FeatureRowExtended;
5050

5151
@AllArgsConstructor
5252
@Slf4j
5353
public class SplitOutputByStore extends PTransform<PFeatureRows, PFeatureRows> {
54+
5455
private Collection<? extends FeatureStore> stores;
5556
private SerializableFunction<FeatureSpec, String> selector;
5657
private Specs specs;
5758

5859
@Override
5960
public PFeatureRows expand(PFeatureRows input) {
6061
Map<String, Write> transforms = getFeatureStoreTransforms();
62+
transforms.put("", new NoOpIO.Write());
6163
Set<String> keys = transforms.keySet();
6264
Preconditions.checkArgument(transforms.size() > 0, "no write transforms found");
6365

@@ -102,6 +104,7 @@ private Map<String, Write> getFeatureStoreTransforms() {
102104

103105
@AllArgsConstructor
104106
public static class WriteTags extends PTransform<PCollectionTuple, PFeatureRows> {
107+
105108
private Map<TupleTag<FeatureRowExtended>, Write> transforms;
106109
private TupleTag<FeatureRowExtended> mainTag;
107110

@@ -120,9 +123,7 @@ public PFeatureRows expand(PCollectionTuple input) {
120123
}
121124

122125
String message =
123-
"FeatureRow with output tag.no matching storage, these feature's "
124-
+ "specs may be specifying a store which was unknown when "
125-
+ "ingestion started as they somehow passed validation. ";
126+
"FeatureRows have no matching write transform, these rows should not have passed validation.";
126127
PCollection<FeatureRowExtended> errors =
127128
input.get(mainTag).apply(ParDo.of(new WithErrors(getName(), message)));
128129

@@ -131,8 +132,11 @@ public PFeatureRows expand(PCollectionTuple input) {
131132
}
132133
}
133134

134-
/** Sets the last attempt error for all rows with a given exception */
135+
/**
136+
* Sets the last attempt error for all rows with a given exception
137+
*/
135138
public static class WithErrors extends DoFn<FeatureRowExtended, FeatureRowExtended> {
139+
136140
private Error error;
137141

138142
public WithErrors(Error error) {

ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,17 +58,18 @@
5858
import org.apache.beam.sdk.transforms.Flatten;
5959
import org.apache.beam.sdk.values.PCollection;
6060
import org.apache.beam.sdk.values.PCollectionList;
61-
import org.junit.Assert;
6261
import org.junit.Rule;
6362
import org.junit.Test;
6463
import org.junit.rules.TemporaryFolder;
6564

6665
@Slf4j
6766
public class ImportJobCSVTest {
6867

69-
@Rule public TemporaryFolder folder = new TemporaryFolder();
68+
@Rule
69+
public TemporaryFolder folder = new TemporaryFolder();
7070

71-
@Rule public TestPipeline testPipeline = TestPipeline.create();
71+
@Rule
72+
public TestPipeline testPipeline = TestPipeline.create();
7273

7374
public ImportSpec initImportSpec(ImportSpec importSpec, String dataFile) throws IOException {
7475
return importSpec.toBuilder().putOptions("path", dataFile).build();
@@ -123,7 +124,7 @@ public void testImportCSV() throws IOException {
123124

124125
PCollection<FeatureRowExtended> writtenToWarehouse =
125126
PCollectionList.of(
126-
WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs())
127+
WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs())
127128
.apply("flatten warehouse input", Flatten.pCollections());
128129

129130
PCollection<FeatureRowExtended> writtenToErrors =
@@ -188,7 +189,7 @@ public void testImportCSVUnknownServingStoreError() throws IOException {
188189
+ " fields:\n"
189190
+ " - name: id\n"
190191
+ " - featureId: testEntity.none.redisInt32\n" // Redis is not available by
191-
// default from the json specs
192+
// default from the json specs
192193
+ " - featureId: testEntity.none.testString\n"
193194
+ "\n",
194195
ImportSpec.getDefaultInstance());

0 commit comments

Comments
 (0)