Skip to content

Commit b5e19bd

Browse files
timsfeast-ci-bot
tims
authored and
feast-ci-bot
committed
move read transforms into "source" package as FeatureSources (#74)
* move read transforms into "source" package as FeatureSources * fix core import spec validator to check for file.csv and file.json * update import spec validator tests
1 parent 9e2de1b commit b5e19bd

32 files changed

Lines changed: 704 additions & 673 deletions

‎core/src/main/java/feast/core/validators/SpecValidator.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException
224224
case "pubsub":
225225
checkPubSubImportSpecOption(spec);
226226
break;
227-
case "file":
227+
case "file.csv":
228+
case "file.json":
228229
checkFileImportSpecOption(spec);
229230
checkArgument(
230231
!spec.getSchema().getEntityIdColumn().equals(""),
@@ -263,9 +264,6 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException
263264

264265
private void checkFileImportSpecOption(ImportSpec spec) throws IllegalArgumentException {
265266
try {
266-
checkArgument(
267-
Lists.newArrayList("json", "csv").contains(spec.getOptionsOrThrow("format")),
268-
"File format must be of type 'json' or 'csv'");
269267
checkArgument(!spec.getOptionsOrDefault("path", "").equals(""), "File path cannot be empty");
270268
} catch (NullPointerException | IllegalArgumentException e) {
271269
throw new IllegalArgumentException(

‎core/src/test/java/feast/core/validators/SpecValidatorTest.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -670,10 +670,10 @@ public void fileImportSpecWithoutSupportedFileFormatShouldThrowIllegalArgumentEx
670670
featureGroupInfoRepository,
671671
featureInfoRepository);
672672
ImportSpec input =
673-
ImportSpec.newBuilder().setType("file").putOptions("format", "notSupported").build();
673+
ImportSpec.newBuilder().setType("file.wat?").build();
674674
exception.expect(IllegalArgumentException.class);
675675
exception.expectMessage(
676-
"Validation for import spec failed: Invalid options: File format must be of type 'json' or 'csv'");
676+
"Validation for import spec failed: Type file.wat? not supported");
677677
validator.validateImportSpec(input);
678678
}
679679

@@ -685,7 +685,7 @@ public void fileImportSpecWithoutValidPathShouldThrowIllegalArgumentException()
685685
entityInfoRepository,
686686
featureGroupInfoRepository,
687687
featureInfoRepository);
688-
ImportSpec input = ImportSpec.newBuilder().setType("file").putOptions("format", "csv").build();
688+
ImportSpec input = ImportSpec.newBuilder().setType("file.csv").build();
689689
exception.expect(IllegalArgumentException.class);
690690
exception.expectMessage(
691691
"Validation for import spec failed: Invalid options: File path cannot be empty");
@@ -702,8 +702,7 @@ public void fileImportSpecWithoutEntityIdColumnInSchemaShouldThrowIllegalArgumen
702702
featureInfoRepository);
703703
ImportSpec input =
704704
ImportSpec.newBuilder()
705-
.setType("file")
706-
.putOptions("format", "csv")
705+
.setType("file.csv")
707706
.putOptions("path", "gs://asdasd")
708707
.build();
709708
exception.expect(IllegalArgumentException.class);

‎ingestion/src/main/java/feast/ingestion/transform/FeatureRowCsvIO.java‎

Lines changed: 0 additions & 135 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/FeatureRowJsonTextIO.java‎

Lines changed: 0 additions & 78 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/FeatureRowKafkaIO.java‎

Lines changed: 0 additions & 97 deletions
This file was deleted.

0 commit comments

Comments
 (0)