From cefa62f9b69f34270b678767bfa05bda81434b30 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 12 Oct 2020 17:12:55 +0800 Subject: [PATCH 01/24] Proto: Add DataFormat proto and use when specifying data format/options Signed-off-by: Zhu Zhanyan --- protos/feast/core/DataSource.proto | 44 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index c3c0f3fc908..6148c15bcb4 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -22,6 +22,33 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; option java_outer_classname = "DataSourceProto"; option java_package = "feast.proto.core"; +// Defines the data format and format specific options +message DataFormat { + // Defines options for the protobuf data format + message ProtoFormat { + // Classpath to the generated Java Protobuf class that can be used to decode + // Feature data from the obtained stream message + string class_path = 1; + } + + // Defines options for the avro data format + message AvroFormat { + // Optional if used in a File DataSource as schema is embedded in avro file. + // Specifies the schema of the Avro message as JSON string. + string schema_json = 1; + } + + // Defines options for the Parquet data format + message ParquetFormat {} + + // Specifies the data format and format specific options + oneof format { + ProtoFormat proto_format = 1; + AvroFormat avro_format = 2; + ParquetFormat parquet_format = 3; + } +} + // Defines a Data Source that can be used source Feature data message DataSource { // Type of Data Source. @@ -50,8 +77,9 @@ message DataSource { // Defines options for DataSource that sources features from a file message FileOptions { - // File Format of the file containing the features - string file_format = 1; + // Defines the file format encoding the features/entity data + // File Data Sources support Avro and Parquet as data formats. + DataFormat file_format = 1; // Target URL of file to retrieve and source features from. // s3://path/to/file for AWS S3 storage @@ -76,9 +104,9 @@ message DataSource { // Kafka topic to collect feature data from. string topic = 2; - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained Kafka message - string class_path = 3; + // Defines the data format encoding feature/entity data in Kafka messages. + // Kafka Data Sources support Avro and Proto as data formats. + DataFormat message_format = 3; } // Defines options for DataSource that sources features from Kinesis records. @@ -91,9 +119,9 @@ message DataSource { // Name of the Kinesis stream to obtain feature data from. string stream_name = 2; - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained Kinesis record - string class_path = 3; + // Defines the data format encoding the feature/entity data in Kinesis records. + // Kinesis Data Sources support Avro and Proto as data formats. + DataFormat record_format = 3; } // DataSource options. From bd64202c0f3b66a7b30daa04c0fa1302cdc3cda8 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 12 Oct 2020 18:41:44 +0800 Subject: [PATCH 02/24] Core: Update DataSource model to support new DataFormat proto Signed-off-by: Zhu Zhanyan --- .../java/feast/common/it/DataGenerator.java | 16 ++++- .../java/feast/core/model/DataSource.java | 72 ++++++++++++------- .../java/feast/core/model/DataSourceTest.java | 8 ++- 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 487455814ab..adb7fbf7e4b 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -20,6 +20,9 @@ import com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; +import feast.proto.core.DataSourceProto.DataFormat; +import feast.proto.core.DataSourceProto.DataFormat.ParquetFormat; +import feast.proto.core.DataSourceProto.DataFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; @@ -270,7 +273,13 @@ public static DataSource createFileDataSourceSpec( return DataSource.newBuilder() .setType(DataSource.SourceType.BATCH_FILE) .setFileOptions( - FileOptions.newBuilder().setFileFormat(fileFormat).setFileUrl(fileURL).build()) + FileOptions.newBuilder() + .setFileFormat( + DataFormat.newBuilder() + .setParquetFormat(ParquetFormat.getDefaultInstance()) + .build()) + .setFileUrl(fileURL) + .build()) .setEventTimestampColumn(timestampColumn) .setDatePartitionColumn(datePartitionColumn) .build(); @@ -294,7 +303,10 @@ public static DataSource createKafkaDataSourceSpec( KafkaOptions.newBuilder() .setTopic(topic) .setBootstrapServers(servers) - .setClassPath(classPath) + .setMessageFormat( + DataFormat.newBuilder() + .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build()) + .build()) .build()) .setEventTimestampColumn(timestampColumn) .build(); diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java index 6bfb5f03033..91513e21607 100644 --- a/core/src/main/java/feast/core/model/DataSource.java +++ b/core/src/main/java/feast/core/model/DataSource.java @@ -16,10 +16,11 @@ */ package feast.core.model; -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; - +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; import feast.core.util.TypeConversion; import feast.proto.core.DataSourceProto; +import feast.proto.core.DataSourceProto.DataFormat; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; @@ -88,27 +89,35 @@ public static DataSource fromProto(DataSourceProto.DataSource spec) { DataSource source = new DataSource(spec.getType()); // Copy source type specific options Map dataSourceConfigMap = new HashMap<>(); - switch (spec.getType()) { - case BATCH_FILE: - dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); - dataSourceConfigMap.put("file_format", spec.getFileOptions().getFileFormat()); - break; - case BATCH_BIGQUERY: - dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); - break; - case STREAM_KAFKA: - dataSourceConfigMap.put("bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); - dataSourceConfigMap.put("class_path", spec.getKafkaOptions().getClassPath()); - dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); - break; - case STREAM_KINESIS: - dataSourceConfigMap.put("class_path", spec.getKinesisOptions().getClassPath()); - dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); - dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); + JsonFormat.Printer json = JsonFormat.printer(); + try { + switch (spec.getType()) { + case BATCH_FILE: + dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); + dataSourceConfigMap.put("file_format", json.print(spec.getFileOptions().getFileFormat())); + break; + case BATCH_BIGQUERY: + dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); + break; + case STREAM_KAFKA: + dataSourceConfigMap.put( + "bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); + dataSourceConfigMap.put( + "message_format", json.print(spec.getKafkaOptions().getMessageFormat())); + dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); + break; + case STREAM_KINESIS: + dataSourceConfigMap.put( + "record_format", json.print(spec.getKinesisOptions().getRecordFormat())); + dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); + dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); + break; + default: + throw new UnsupportedOperationException( + String.format("Unsupported Feature Store Type: %s", spec.getType())); + } + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException("Unexpected exception convering DataFormat Proto to JSON", e); } // Store DataSource mapping as serialised JSON @@ -137,7 +146,7 @@ public DataSourceProto.DataSource toProto() { case BATCH_FILE: FileOptions.Builder fileOptions = FileOptions.newBuilder(); fileOptions.setFileUrl(dataSourceConfigMap.get("file_url")); - fileOptions.setFileFormat(dataSourceConfigMap.get("file_format")); + fileOptions.setFileFormat(parseFormat(dataSourceConfigMap.get("file_format"))); spec.setFileOptions(fileOptions.build()); break; case BATCH_BIGQUERY: @@ -148,15 +157,15 @@ public DataSourceProto.DataSource toProto() { case STREAM_KAFKA: KafkaOptions.Builder kafkaOptions = KafkaOptions.newBuilder(); kafkaOptions.setBootstrapServers(dataSourceConfigMap.get("bootstrap_servers")); - kafkaOptions.setClassPath(dataSourceConfigMap.get("class_path")); kafkaOptions.setTopic(dataSourceConfigMap.get("topic")); + kafkaOptions.setMessageFormat(parseFormat(dataSourceConfigMap.get("message_format"))); spec.setKafkaOptions(kafkaOptions.build()); break; case STREAM_KINESIS: KinesisOptions.Builder kinesisOptions = KinesisOptions.newBuilder(); - kinesisOptions.setClassPath(dataSourceConfigMap.get("class_path")); kinesisOptions.setRegion(dataSourceConfigMap.get("region")); kinesisOptions.setStreamName(dataSourceConfigMap.get("stream_name")); + kinesisOptions.setRecordFormat(parseFormat(dataSourceConfigMap.get("record_format"))); spec.setKinesisOptions(kinesisOptions.build()); break; default: @@ -194,4 +203,15 @@ public boolean equals(Object o) { DataSource other = (DataSource) o; return this.toProto().equals(other.toProto()); } + + /** Parse the given data format in JSON representation to its protobuf representation. */ + private DataFormat parseFormat(String formatJSON) { + try { + DataFormat.Builder format = DataFormat.newBuilder(); + JsonFormat.parser().merge(formatJSON, format); + return format.build(); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException("Unexpected exception convering DataFormat JSON to Proto", e); + } + } } diff --git a/core/src/test/java/feast/core/model/DataSourceTest.java b/core/src/test/java/feast/core/model/DataSourceTest.java index 8e7400dc8b2..59f3738ba9b 100644 --- a/core/src/test/java/feast/core/model/DataSourceTest.java +++ b/core/src/test/java/feast/core/model/DataSourceTest.java @@ -22,6 +22,8 @@ import feast.common.it.DataGenerator; import feast.proto.core.DataSourceProto; +import feast.proto.core.DataSourceProto.DataFormat; +import feast.proto.core.DataSourceProto.DataFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import java.util.List; @@ -68,7 +70,11 @@ private List getTestSpecs() { KinesisOptions.newBuilder() .setRegion("ap-nowhere1") .setStreamName("stream") - .setClassPath("class.path") + .setRecordFormat( + DataFormat.newBuilder() + .setProtoFormat( + ProtoFormat.newBuilder().setClassPath("class.path").build()) + .build()) .build()) .build()); } From 8697e61cbb99d7c797bec657c8e104a3b66ade12 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 13 Oct 2020 19:09:16 +0800 Subject: [PATCH 03/24] Python SDK: Update SDK to use new data format API. Signed-off-by: Zhu Zhanyan --- sdk/python/feast/client.py | 6 +- sdk/python/feast/data_source.py | 160 +++++++++++++++++++++++--------- sdk/python/tests/test_client.py | 15 ++- 3 files changed, 129 insertions(+), 52 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index ed80714845c..b13a3e53b96 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -657,10 +657,8 @@ def ingest( if ( feature_table.batch_source and issubclass(type(feature_table.batch_source), FileSource) - and "".join( - feature_table.batch_source.file_options.file_format.split() - ).lower() - != "parquet" + and feature_table.batch_source.file_options.file_format.WhichOneof("format") + != "parquet_format" ): raise Exception( f"No suitable batch source found for FeatureTable, {name}." diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 421d3d5c5b7..86810c719c5 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,8 +14,10 @@ import enum +from abc import ABC, abstractclassmethod, abstractmethod from typing import Dict, Optional +from feast.core.DataSource_pb2 import DataFormat as DataFormatProto from feast.core.DataSource_pb2 import DataSource as DataSourceProto @@ -31,13 +33,87 @@ class SourceType(enum.Enum): STREAM_KINESIS = 4 +class DataFormat(ABC): + """ + Defines an abstract data format used to encode feature data. + """ + + @abstractmethod + def to_proto(self): + """ + Convert this DataFormat into its protobuf representation. + """ + pass + + @classmethod + def from_proto(cls, proto): + """ + Construct this DataFormat from its protobuf representation. + """ + fmt = proto.WhichOneof("format") + if fmt == "avro_format": + return AvroFormat(schema_json=proto.avro_format.schema_json) + if fmt == "parquet_format": + return ParquetFormat() + if fmt == "proto_format": + return ProtoFormat(class_path=proto.proto_format.class_path) + + +class AvroFormat(DataFormat): + """ + Defines the Avro data format + """ + + def __init__(self, schema_json: str): + """ + Construct a new Avro data format. + + Args: + schema_json: Avro schema definition in JSON + """ + self.schema_json = schema_json + + def to_proto(self): + proto = DataFormatProto.AvroFormat(schema_json=self.schema_json) + return DataFormatProto(avro_format=proto) + + +class ParquetFormat(DataFormat): + """ + Defines the Parquet data format + """ + + def to_proto(self): + return DataFormatProto(parquet_format=DataFormatProto.ParquetFormat()) + + +class ProtoFormat(DataFormat): + """ + Defines the Protobuf data format + """ + + def __init__(self, class_path: str): + """ + Construct a new Avro data format. + + Args: + schema_json: Avro schema definition in JSON + """ + self.class_path = class_path + + def to_proto(self): + return DataFormatProto( + proto_format=DataFormatProto.ProtoFormat(class_path=self.class_path) + ) + + class FileOptions: """ DataSource File options used to source features from a file """ def __init__( - self, file_format: str, file_url: str, + self, file_format: DataFormat, file_url: str, ): self._file_format = file_format self._file_url = file_url @@ -75,18 +151,16 @@ def from_proto(cls, file_options_proto: DataSourceProto.FileOptions): """ Creates a FileOptions from a protobuf representation of a file option - Args: - file_options_proto: A protobuf representation of a DataSource + args: + file_options_proto: a protobuf representation of a datasource Returns: Returns a FileOptions object based on the file_options protobuf """ - file_options = cls( - file_format=file_options_proto.file_format, + file_format=DataFormat.from_proto(file_options_proto.file_format), file_url=file_options_proto.file_url, ) - return file_options def to_proto(self) -> DataSourceProto.FileOptions: @@ -98,7 +172,7 @@ def to_proto(self) -> DataSourceProto.FileOptions: """ file_options_proto = DataSourceProto.FileOptions( - file_format=self.file_format, file_url=self.file_url, + file_format=self.file_format.to_proto(), file_url=self.file_url, ) return file_options_proto @@ -165,10 +239,10 @@ class KafkaOptions: """ def __init__( - self, bootstrap_servers: str, class_path: str, topic: str, + self, bootstrap_servers: str, message_format: DataFormat, topic: str, ): self._bootstrap_servers = bootstrap_servers - self._class_path = class_path + self._message_format = message_format self._topic = topic @property @@ -186,20 +260,18 @@ def bootstrap_servers(self, bootstrap_servers): self._bootstrap_servers = bootstrap_servers @property - def class_path(self): + def message_format(self): """ - Returns the class path to the generated Java Protobuf class that can be - used to decode feature data from the obtained Kafka message + Returns the data format that is used to encode the feature data in Kafka messages """ - return self._class_path + return self._message_format - @class_path.setter - def class_path(self, class_path): + @message_format.setter + def message_format(self, message_format): """ - Sets the class path to the generated Java Protobuf class that can be - used to decode feature data from the obtained Kafka message + Sets the data format that is used to encode the feature data in Kafka messages """ - self._class_path = class_path + self._message_format = message_format @property def topic(self): @@ -229,7 +301,7 @@ def from_proto(cls, kafka_options_proto: DataSourceProto.KafkaOptions): kafka_options = cls( bootstrap_servers=kafka_options_proto.bootstrap_servers, - class_path=kafka_options_proto.class_path, + message_format=DataFormat.from_proto(kafka_options_proto.message_format), topic=kafka_options_proto.topic, ) @@ -245,7 +317,7 @@ def to_proto(self) -> DataSourceProto.KafkaOptions: kafka_options_proto = DataSourceProto.KafkaOptions( bootstrap_servers=self.bootstrap_servers, - class_path=self.class_path, + message_format=self.message_format.to_proto(), topic=self.topic, ) @@ -258,27 +330,25 @@ class KinesisOptions: """ def __init__( - self, class_path: str, region: str, stream_name: str, + self, record_format: DataFormat, region: str, stream_name: str, ): - self._class_path = class_path + self._record_format = record_format self._region = region self._stream_name = stream_name @property - def class_path(self): + def record_format(self): """ - Returns the class path to the generated Java Protobuf class that can be - used to decode feature data from the obtained Kinesis record + Returns the data format used to encode the feature data in the Kinesis records. """ - return self._class_path + return self._record_format - @class_path.setter - def class_path(self, class_path): + @record_format.setter + def record_format(self, record_format): """ - Sets the class path to the generated Java Protobuf class that can be - used to decode feature data from the obtained Kinesis record + Sets the data format used to encode the feature data in the Kinesis records. """ - self._class_path = class_path + self._record_format = record_format @property def region(self): @@ -321,7 +391,7 @@ def from_proto(cls, kinesis_options_proto: DataSourceProto.KinesisOptions): """ kinesis_options = cls( - class_path=kinesis_options_proto.class_path, + record_format=DataFormat.from_proto(kinesis_options_proto.record_format), region=kinesis_options_proto.region, stream_name=kinesis_options_proto.stream_name, ) @@ -337,7 +407,7 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: """ kinesis_options_proto = DataSourceProto.KinesisOptions( - class_path=self.class_path, + record_format=self.record_format.to_proto(), region=self.region, stream_name=self.stream_name, ) @@ -458,25 +528,25 @@ def from_proto(data_source): elif ( data_source.kafka_options.bootstrap_servers and data_source.kafka_options.topic - and data_source.kafka_options.class_path + and data_source.kafka_options.message_format ): data_source_obj = KafkaSource( field_mapping=data_source.field_mapping, bootstrap_servers=data_source.kafka_options.bootstrap_servers, - class_path=data_source.kafka_options.class_path, + message_format=data_source.kafka_options.message_format, topic=data_source.kafka_options.topic, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, date_partition_column=data_source.date_partition_column, ) elif ( - data_source.kinesis_options.class_path + data_source.kinesis_options.record_format and data_source.kinesis_options.region and data_source.kinesis_options.stream_name ): data_source_obj = KinesisSource( field_mapping=data_source.field_mapping, - class_path=data_source.kinesis_options.class_path, + record_format=data_source.kinesis_options.record_format, region=data_source.kinesis_options.region, stream_name=data_source.kinesis_options.stream_name, event_timestamp_column=data_source.event_timestamp_column, @@ -500,7 +570,7 @@ def __init__( self, event_timestamp_column: str, created_timestamp_column: str, - file_format: str, + file_format: DataFormat, file_url: str, field_mapping: Optional[Dict[str, str]] = dict(), date_partition_column: Optional[str] = "", @@ -615,7 +685,7 @@ def __init__( event_timestamp_column: str, created_timestamp_column: str, bootstrap_servers: str, - class_path: str, + message_format: DataFormat, topic: str, field_mapping: Optional[Dict[str, str]] = dict(), date_partition_column: Optional[str] = "", @@ -627,7 +697,9 @@ def __init__( date_partition_column, ) self._kafka_options = KafkaOptions( - bootstrap_servers=bootstrap_servers, class_path=class_path, topic=topic + bootstrap_servers=bootstrap_servers, + message_format=message_format, + topic=topic, ) def __eq__(self, other): @@ -639,7 +711,7 @@ def __eq__(self, other): if ( self.kafka_options.bootstrap_servers != other.kafka_options.bootstrap_servers - or self.kafka_options.class_path != other.kafka_options.class_path + or self.kafka_options.message_format != other.kafka_options.message_format or self.kafka_options.topic != other.kafka_options.topic ): return False @@ -679,7 +751,7 @@ def __init__( self, event_timestamp_column: str, created_timestamp_column: str, - class_path: str, + record_format: DataFormat, region: str, stream_name: str, field_mapping: Optional[Dict[str, str]] = dict(), @@ -692,7 +764,7 @@ def __init__( date_partition_column, ) self._kinesis_options = KinesisOptions( - class_path=class_path, region=region, stream_name=stream_name + record_format=record_format, region=region, stream_name=stream_name ) def __eq__(self, other): @@ -702,7 +774,7 @@ def __eq__(self, other): ) if ( - self.kinesis_options.class_path != other.kinesis_options.class_path + self.kinesis_options.record_format != other.kinesis_options.record_format or self.kinesis_options.region != other.kinesis_options.region or self.kinesis_options.stream_name != other.kinesis_options.stream_name ): diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 1d970907d50..ecba7224803 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -41,7 +41,14 @@ from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto from feast.core.FeatureTable_pb2 import FeatureTableMeta as FeatureTableMetaProto from feast.core.FeatureTable_pb2 import FeatureTableSpec as FeatureTableSpecProto -from feast.data_source import FileSource, KafkaSource +from feast.data_source import ( + AvroFormat, + DataFormat, + FileSource, + KafkaSource, + ParquetFormat, + ProtoFormat, +) from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable @@ -398,7 +405,7 @@ def test_apply_feature_table_success(self, test_client): # Create Feature Tables batch_source = FileSource( - file_format="parquet", + file_format=ParquetFormat(), file_url="file://feast/*", event_timestamp_column="ts_col", created_timestamp_column="timestamp", @@ -407,7 +414,7 @@ def test_apply_feature_table_success(self, test_client): stream_source = KafkaSource( bootstrap_servers="localhost:9094", - class_path="random/path/to/class", + message_format=ProtoFormat("class.path"), topic="test_topic", event_timestamp_column="ts_col", created_timestamp_column="timestamp", @@ -820,7 +827,7 @@ def _ingest_test_getfeaturetable_mocked_resp( entities=["dev_entity"], batch_source=DataSourceProto( file_options=DataSourceProto.FileOptions( - file_format="parquet", file_url=file_url + file_format=ParquetFormat().to_proto(), file_url=file_url ), event_timestamp_column="datetime", created_timestamp_column="timestamp", From 977f4e4f7194387ad2c71457bd574bd57f208146 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 13 Oct 2020 19:32:35 +0800 Subject: [PATCH 04/24] Fix python lint Signed-off-by: Zhu Zhanyan --- sdk/python/feast/data_source.py | 2 +- sdk/python/tests/test_client.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 86810c719c5..166602d99ab 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,7 +14,7 @@ import enum -from abc import ABC, abstractclassmethod, abstractmethod +from abc import ABC, abstractmethod from typing import Dict, Optional from feast.core.DataSource_pb2 import DataFormat as DataFormatProto diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index ecba7224803..147a8bbe7cd 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -41,14 +41,7 @@ from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto from feast.core.FeatureTable_pb2 import FeatureTableMeta as FeatureTableMetaProto from feast.core.FeatureTable_pb2 import FeatureTableSpec as FeatureTableSpecProto -from feast.data_source import ( - AvroFormat, - DataFormat, - FileSource, - KafkaSource, - ParquetFormat, - ProtoFormat, -) +from feast.data_source import FileSource, KafkaSource, ParquetFormat, ProtoFormat from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable From 4fa2e0aede8e55fc309c6d63ffefdf64efec8e50 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 13 Oct 2020 22:07:21 +0800 Subject: [PATCH 05/24] Fix Python unit tests refercing old argument `class_path' Signed-off-by: Zhu Zhanyan --- sdk/python/tests/test_feature_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/test_feature_table.py b/sdk/python/tests/test_feature_table.py index 0dd8bb17172..0330ae6158e 100644 --- a/sdk/python/tests/test_feature_table.py +++ b/sdk/python/tests/test_feature_table.py @@ -21,7 +21,7 @@ from feast.client import Client from feast.core import CoreService_pb2_grpc as Core -from feast.data_source import FileSource, KafkaSource +from feast.data_source import FileSource, KafkaSource, ParquetFormat, ProtoFormat from feast.feature import Feature from feast.feature_table import FeatureTable from feast.value_type import ValueType @@ -59,7 +59,7 @@ def test_feature_table_import_export_yaml(self): "ride_distance": "ride_distance", "ride_duration": "ride_duration", }, - file_format="parquet", + file_format=ParquetFormat(), file_url="file://feast/*", event_timestamp_column="ts_col", created_timestamp_column="timestamp", @@ -72,7 +72,7 @@ def test_feature_table_import_export_yaml(self): "ride_duration": "ride_duration", }, bootstrap_servers="localhost:9094", - class_path="random/path/to/class", + message_format=ProtoFormat(class_path="class.path"), topic="test_topic", event_timestamp_column="ts_col", created_timestamp_column="timestamp", From 395e1a576eac07bb598fb626ae61b13dc3c7246c Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 11:09:53 +0800 Subject: [PATCH 06/24] Fix DataSource's from_proto() not coverting DataFormat protos to native objects. Signed-off-by: Zhu Zhanyan --- sdk/python/feast/client.py | 6 +++--- sdk/python/feast/data_source.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index b13a3e53b96..89be4fd0ea1 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -58,7 +58,7 @@ ListProjectsResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.data_source import BigQuerySource, FileSource +from feast.data_source import BigQuerySource, FileSource, ParquetFormat from feast.entity import Entity from feast.feature import _build_feature_references from feast.feature_table import FeatureTable @@ -87,6 +87,7 @@ ) from feast.serving.ServingService_pb2_grpc import ServingServiceStub + _logger = logging.getLogger(__name__) CPU_COUNT: int = multiprocessing.cpu_count() @@ -657,8 +658,7 @@ def ingest( if ( feature_table.batch_source and issubclass(type(feature_table.batch_source), FileSource) - and feature_table.batch_source.file_options.file_format.WhichOneof("format") - != "parquet_format" + and isinstance(type(feature_table.batch_source.file_options.file_format), ParquetFormat) ): raise Exception( f"No suitable batch source found for FeatureTable, {name}." diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 166602d99ab..a4aa0d67cb3 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -45,6 +45,9 @@ def to_proto(self): """ pass + def __eq__(self, other): + return self.to_proto() == other.to_proto() + @classmethod def from_proto(cls, proto): """ @@ -94,10 +97,10 @@ class ProtoFormat(DataFormat): def __init__(self, class_path: str): """ - Construct a new Avro data format. + Construct a new Protobuf data format. Args: - schema_json: Avro schema definition in JSON + class_path: Class path to the Java Protobuf class that can be used to decode protobuf messages.; """ self.class_path = class_path @@ -106,7 +109,6 @@ def to_proto(self): proto_format=DataFormatProto.ProtoFormat(class_path=self.class_path) ) - class FileOptions: """ DataSource File options used to source features from a file @@ -511,7 +513,7 @@ def from_proto(data_source): if data_source.file_options.file_format and data_source.file_options.file_url: data_source_obj = FileSource( field_mapping=data_source.field_mapping, - file_format=data_source.file_options.file_format, + file_format=DataFormat.from_proto(data_source.file_options.file_format), file_url=data_source.file_options.file_url, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, @@ -533,7 +535,7 @@ def from_proto(data_source): data_source_obj = KafkaSource( field_mapping=data_source.field_mapping, bootstrap_servers=data_source.kafka_options.bootstrap_servers, - message_format=data_source.kafka_options.message_format, + message_format=DataFormat.from_proto(data_source.kafka_options.message_format), topic=data_source.kafka_options.topic, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, @@ -546,7 +548,7 @@ def from_proto(data_source): ): data_source_obj = KinesisSource( field_mapping=data_source.field_mapping, - record_format=data_source.kinesis_options.record_format, + record_format=DataFormat.from_proto(data_source.kinesis_options.record_format), region=data_source.kinesis_options.region, stream_name=data_source.kinesis_options.stream_name, event_timestamp_column=data_source.event_timestamp_column, @@ -592,7 +594,6 @@ def __eq__(self, other): or self.file_options.file_format != other.file_options.file_format ): return False - return True @property From 07548997e0436f8baeb17de5b1ee45823e40a60c Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 11:31:46 +0800 Subject: [PATCH 07/24] Fix python lint Signed-off-by: Zhu Zhanyan --- sdk/python/feast/client.py | 5 +++-- sdk/python/feast/data_source.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 89be4fd0ea1..0b73c6ad459 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -87,7 +87,6 @@ ) from feast.serving.ServingService_pb2_grpc import ServingServiceStub - _logger = logging.getLogger(__name__) CPU_COUNT: int = multiprocessing.cpu_count() @@ -658,7 +657,9 @@ def ingest( if ( feature_table.batch_source and issubclass(type(feature_table.batch_source), FileSource) - and isinstance(type(feature_table.batch_source.file_options.file_format), ParquetFormat) + and isinstance( + type(feature_table.batch_source.file_options.file_format), ParquetFormat + ) ): raise Exception( f"No suitable batch source found for FeatureTable, {name}." diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index a4aa0d67cb3..a3ed485f904 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -109,6 +109,7 @@ def to_proto(self): proto_format=DataFormatProto.ProtoFormat(class_path=self.class_path) ) + class FileOptions: """ DataSource File options used to source features from a file @@ -535,7 +536,9 @@ def from_proto(data_source): data_source_obj = KafkaSource( field_mapping=data_source.field_mapping, bootstrap_servers=data_source.kafka_options.bootstrap_servers, - message_format=DataFormat.from_proto(data_source.kafka_options.message_format), + message_format=DataFormat.from_proto( + data_source.kafka_options.message_format + ), topic=data_source.kafka_options.topic, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, @@ -548,7 +551,9 @@ def from_proto(data_source): ): data_source_obj = KinesisSource( field_mapping=data_source.field_mapping, - record_format=DataFormat.from_proto(data_source.kinesis_options.record_format), + record_format=DataFormat.from_proto( + data_source.kinesis_options.record_format + ), region=data_source.kinesis_options.region, stream_name=data_source.kinesis_options.stream_name, event_timestamp_column=data_source.event_timestamp_column, From aa2bdfa4ee594703a287f94d12150d662efb686c Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 15:06:43 +0800 Subject: [PATCH 08/24] Core: Add convience methods to DataGenerator to create Kinesis source, DataFormat specs Signed-off-by: Zhu Zhanyan --- .../java/feast/common/it/DataGenerator.java | 40 +++++++++++++++---- .../java/feast/core/model/DataSourceTest.java | 25 +----------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index adb7fbf7e4b..fa4df05c4ee 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -21,12 +21,14 @@ import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; import feast.proto.core.DataSourceProto.DataFormat; +import feast.proto.core.DataSourceProto.DataFormat.AvroFormat; import feast.proto.core.DataSourceProto.DataFormat.ParquetFormat; import feast.proto.core.DataSourceProto.DataFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; +import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import feast.proto.core.EntityProto; import feast.proto.core.FeatureProto; import feast.proto.core.FeatureProto.FeatureSpecV2; @@ -274,10 +276,7 @@ public static DataSource createFileDataSourceSpec( .setType(DataSource.SourceType.BATCH_FILE) .setFileOptions( FileOptions.newBuilder() - .setFileFormat( - DataFormat.newBuilder() - .setParquetFormat(ParquetFormat.getDefaultInstance()) - .build()) + .setFileFormat(createParquetFormat()) .setFileUrl(fileURL) .build()) .setEventTimestampColumn(timestampColumn) @@ -303,10 +302,7 @@ public static DataSource createKafkaDataSourceSpec( KafkaOptions.newBuilder() .setTopic(topic) .setBootstrapServers(servers) - .setMessageFormat( - DataFormat.newBuilder() - .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build()) - .build()) + .setMessageFormat(createProtoFormat("class.path")) .build()) .setEventTimestampColumn(timestampColumn) .build(); @@ -337,6 +333,34 @@ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityR return ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() .setTimestamp(Timestamp.newBuilder().setSeconds(seconds)) .putFields(entityName, entityValue) + + public static DataSource createKinesisDataSourceSpec( + String region, String streamName, String classPath, String timestampColumn) { + return DataSource.newBuilder() + .setType(DataSource.SourceType.STREAM_KINESIS) + .setKinesisOptions( + KinesisOptions.newBuilder() + .setRegion("ap-nowhere1") + .setStreamName("stream") + .setRecordFormat(createProtoFormat("classPath")) + .build()) + .setEventTimestampColumn(timestampColumn) + .build(); + } + + public static DataFormat createParquetFormat() { + return DataFormat.newBuilder().setParquetFormat(ParquetFormat.getDefaultInstance()).build(); + } + + public static DataFormat createAvroFormat(String schemaJSON) { + return DataFormat.newBuilder() + .setAvroFormat(AvroFormat.newBuilder().setSchemaJson(schemaJSON).build()) + .build(); + } + + public static DataFormat createProtoFormat(String classPath) { + return DataFormat.newBuilder() + .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build()) .build(); } } diff --git a/core/src/test/java/feast/core/model/DataSourceTest.java b/core/src/test/java/feast/core/model/DataSourceTest.java index 59f3738ba9b..20445ecb441 100644 --- a/core/src/test/java/feast/core/model/DataSourceTest.java +++ b/core/src/test/java/feast/core/model/DataSourceTest.java @@ -16,16 +16,11 @@ */ package feast.core.model; -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; import feast.common.it.DataGenerator; import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataFormat; -import feast.proto.core.DataSourceProto.DataFormat.ProtoFormat; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import java.util.List; import java.util.Map; import org.junit.Test; @@ -59,23 +54,7 @@ private List getTestSpecs() { return List.of( DataGenerator.createFileDataSourceSpec("file:///path/to/file", "parquet", "ts_col", ""), DataGenerator.createKafkaDataSourceSpec("localhost:9092", "topic", "class.path", "ts_col"), - DataSourceProto.DataSource.newBuilder() - .setType(BATCH_BIGQUERY) - .setBigqueryOptions( - BigQueryOptions.newBuilder().setTableRef("project:dataset.table").build()) - .build(), - DataSourceProto.DataSource.newBuilder() - .setType(STREAM_KINESIS) - .setKinesisOptions( - KinesisOptions.newBuilder() - .setRegion("ap-nowhere1") - .setStreamName("stream") - .setRecordFormat( - DataFormat.newBuilder() - .setProtoFormat( - ProtoFormat.newBuilder().setClassPath("class.path").build()) - .build()) - .build()) - .build()); + DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), + DataGenerator.createKinesisDataSourceSpec("ap-nowhere1", "stream", "class.path", "ts_col")); } } From 9b328d6f8884afdca1a9c939b6904e128f982a24 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 15:08:24 +0800 Subject: [PATCH 09/24] Core: Add DataSourceValidator to valid data source specs in Feature Tables Signed-off-by: Zhu Zhanyan --- .../core/validators/DataSourceValidator.java | 71 ++++++++++ .../validators/FeatureTableValidator.java | 15 ++- .../feast/core/service/SpecServiceIT.java | 2 +- .../validators/DataSourceValidatorTest.java | 123 ++++++++++++++++++ 4 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/feast/core/validators/DataSourceValidator.java create mode 100644 core/src/test/java/feast/core/validators/DataSourceValidatorTest.java diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java new file mode 100644 index 00000000000..acf4a34e478 --- /dev/null +++ b/core/src/main/java/feast/core/validators/DataSourceValidator.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.validators; + +import static feast.core.validators.Matchers.*; +import static feast.proto.core.DataSourceProto.DataFormat.FormatCase.*; +import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; + +import feast.proto.core.DataSourceProto.DataFormat.FormatCase; +import feast.proto.core.DataSourceProto.DataSource; +import feast.proto.core.DataSourceProto.DataSource.SourceType; +import java.util.Map; +import java.util.Set; + +public class DataSourceValidator { + // Map data source type to its supported data formats + public static final Map> SUPPORTED_FORMATS = + Map.of( + BATCH_FILE, Set.of(PARQUET_FORMAT), + STREAM_KAFKA, Set.of(PROTO_FORMAT), + STREAM_KINESIS, Set.of(PROTO_FORMAT)); + + /** Validate if the given DataSource protobuf spec is valid. */ + public static void validate(DataSource spec) { + switch (spec.getType()) { + case BATCH_FILE: + // Verify that DataFormat is supported by file source + FormatCase fileFormat = spec.getFileOptions().getFileFormat().getFormatCase(); + validateFormat(spec.getType(), fileFormat); + break; + case BATCH_BIGQUERY: + checkValidBigQueryTableRef(spec.getBigqueryOptions().getTableRef(), "FeatureTable"); + break; + case STREAM_KAFKA: + // Verify that DataFormat is supported by kafka data source + FormatCase messageFormat = spec.getKafkaOptions().getMessageFormat().getFormatCase(); + validateFormat(spec.getType(), messageFormat); + break; + case STREAM_KINESIS: + // Verify that DataFormat is supported by kinesis data source + FormatCase recordFormat = spec.getKinesisOptions().getRecordFormat().getFormatCase(); + validateFormat(spec.getType(), recordFormat); + break; + default: + throw new UnsupportedOperationException( + String.format("Unsupported Feature Store Type: %s", spec.getType())); + } + } + + /** Valdiates if the given data format is valid for the given data source type. */ + private static void validateFormat(SourceType sourceType, FormatCase format) { + if (!SUPPORTED_FORMATS.get(sourceType).contains(format)) { + throw new UnsupportedOperationException( + String.format("Unsupported Data Format for %s DataSource: %s", sourceType, format)); + } + } +} diff --git a/core/src/main/java/feast/core/validators/FeatureTableValidator.java b/core/src/main/java/feast/core/validators/FeatureTableValidator.java index d50be2736cd..863c9442f77 100644 --- a/core/src/main/java/feast/core/validators/FeatureTableValidator.java +++ b/core/src/main/java/feast/core/validators/FeatureTableValidator.java @@ -18,6 +18,7 @@ import static feast.core.validators.Matchers.*; +import feast.proto.core.DataSourceProto.DataSource.SourceType; import feast.proto.core.FeatureProto.FeatureSpecV2; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import java.util.ArrayList; @@ -49,12 +50,6 @@ public static void validateSpec(FeatureTableSpec spec) { checkValidCharacters(spec.getName(), "FeatureTable"); spec.getFeaturesList().forEach(FeatureTableValidator::validateFeatureSpec); - // Check that BigQuery reference defined for BigQuery source is valid - if (!spec.getBatchSource().getBigqueryOptions().getTableRef().isEmpty()) { - checkValidBigQueryTableRef( - spec.getBatchSource().getBigqueryOptions().getTableRef(), "FeatureTable"); - } - // Check that features and entities defined in FeatureTable do not use reserved names ArrayList fieldNames = new ArrayList<>(spec.getEntitiesList()); fieldNames.addAll( @@ -70,6 +65,14 @@ public static void validateSpec(FeatureTableSpec spec) { throw new IllegalArgumentException( String.format("Entity and Feature names within a Feature Table should be unique.")); } + + // Check that the data sources defined in the feature table are valid + if (!spec.getBatchSource().getType().equals(SourceType.INVALID)) { + DataSourceValidator.validate(spec.getBatchSource()); + } + if (!spec.getStreamSource().getType().equals(SourceType.INVALID)) { + DataSourceValidator.validate(spec.getStreamSource()); + } } private static void validateFeatureSpec(FeatureSpecV2 spec) { diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 482fa65d834..f4713c8ab7c 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -1213,7 +1213,7 @@ public void shouldErrorOnInvalidBigQueryTableRef() { DataGenerator.createFeatureTableSpec( "ft", List.of("entity1"), - Map.of("event_timestamp", ValueProto.ValueType.Enum.INT64), + Map.of("feature", ValueProto.ValueType.Enum.INT64), 3600, Map.of()) .toBuilder() diff --git a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java new file mode 100644 index 00000000000..d3400379b33 --- /dev/null +++ b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.validators; + +import static feast.proto.core.DataSourceProto.DataFormat.FormatCase.*; +import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; +import static org.junit.Assert.assertTrue; + +import feast.common.it.DataGenerator; +import feast.proto.core.DataSourceProto; +import feast.proto.core.DataSourceProto.DataFormat.FormatCase; +import feast.proto.core.DataSourceProto.DataSource.SourceType; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.Test; + +public class DataSourceValidatorTest { + + @Test(expected = UnsupportedOperationException.class) + public void shouldErrorIfSourceTypeUnsupported() { + DataSourceProto.DataSource badSpec = + getTestSpecsMap().get(BATCH_FILE).toBuilder().setType(SourceType.INVALID).build(); + DataSourceValidator.validate(badSpec); + } + + @Test + public void shouldPassValidSpecs() { + getTestSpecsMap().values().forEach(DataSourceValidator::validate); + } + + public void shouldErrorOnUnsupportedDataFormats() { + + // Invert the supported formats to get a map of unsupported formats + Map> unsupportedFormatMap = + DataSourceValidator.SUPPORTED_FORMATS.entrySet().stream() + .map( + es -> { + Set supportedFormats = es.getValue(); + Set badFormats = + Stream.of(FormatCase.values()) + .filter(format -> !supportedFormats.contains(format)) + .collect(Collectors.toSet()); + return Map.entry(es.getKey(), badFormats); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Check that validator should reject unsupported formats + unsupportedFormatMap.forEach( + (sourceType, badFormats) -> { + DataSourceProto.DataSource.Builder spec = getTestSpecsMap().get(sourceType).toBuilder(); + badFormats.forEach( + badFormat -> { + switch (sourceType) { + case BATCH_FILE: + spec.setFileOptions( + spec.getFileOptions() + .toBuilder() + .setFileFormat(getTestFormatsMap().get(badFormat))); + break; + case STREAM_KAFKA: + spec.setKafkaOptions( + spec.getKafkaOptions() + .toBuilder() + .setMessageFormat(getTestFormatsMap().get(badFormat))); + break; + case STREAM_KINESIS: + spec.setKinesisOptions( + spec.getKinesisOptions() + .toBuilder() + .setRecordFormat(getTestFormatsMap().get(badFormat))); + default: + throw new RuntimeException( + String.format("Unhandled data source type in test: %s", sourceType)); + } + + boolean hasError = false; + try { + DataSourceValidator.validate(spec.build()); + } catch (UnsupportedOperationException e) { + hasError = true; + } + assertTrue(hasError); + }); + }); + } + + private Map getTestSpecsMap() { + return Map.of( + BATCH_FILE, + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "parquet", "ts_col", ""), + BATCH_BIGQUERY, + DataGenerator.createKafkaDataSourceSpec( + "localhost:9092", "topic", "class.path", "ts_col"), + STREAM_KINESIS, + DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), + STREAM_KAFKA, + DataGenerator.createKinesisDataSourceSpec( + "ap-nowhere1", "stream", "class.path", "ts_col")); + } + + private Map getTestFormatsMap() { + return Map.of( + PARQUET_FORMAT, DataGenerator.createParquetFormat(), + PROTO_FORMAT, DataGenerator.createProtoFormat("class.path"), + AVRO_FORMAT, DataGenerator.createAvroFormat("{}")); + } +} From f88cfaabd3475fdf4c1af1f6bd8033634b27e957 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 16:46:22 +0800 Subject: [PATCH 10/24] E2E: Fix specifying data format in FeatureTables in e2e tests Signed-off-by: Zhu Zhanyan --- tests/e2e/test-register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index 1bb64a711bd..edb4333c7c8 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -10,7 +10,7 @@ from pandas.testing import assert_frame_equal from feast.client import Client -from feast.data_source import BigQuerySource, FileSource, KafkaSource +from feast.data_source import BigQuerySource, FileSource, KafkaSource, ParquetFormat, ProtoFormat from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable @@ -67,7 +67,7 @@ def basic_featuretable(): "dev_feature_float": "dev_feature_float_field", "dev_feature_string": "dev_feature_string_field", }, - file_format="PARQUET", + file_format=ParquetFormat(), file_url="gs://example/feast/*", event_timestamp_column="datetime_col", created_timestamp_column="timestamp", @@ -80,7 +80,7 @@ def basic_featuretable(): "dev_feature_string": "dev_feature_string_field", }, bootstrap_servers="localhost:9094", - class_path="random/path/to/class", + message_format=ProtoFormat(class_path="class.path"), topic="test_topic", event_timestamp_column="datetime_col", created_timestamp_column="timestamp", From 4afae3f4d846fe975e3a4aeed72e478759151de1 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 17:01:03 +0800 Subject: [PATCH 11/24] Rebase on master Signed-off-by: Zhu Zhanyan --- .../src/main/java/feast/common/it/DataGenerator.java | 2 ++ tests/e2e/test-register.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index fa4df05c4ee..6f7b37251b2 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -333,6 +333,8 @@ public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityR return ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder() .setTimestamp(Timestamp.newBuilder().setSeconds(seconds)) .putFields(entityName, entityValue) + .build(); + } public static DataSource createKinesisDataSourceSpec( String region, String streamName, String classPath, String timestampColumn) { diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index edb4333c7c8..7d2c8a7c6c7 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -10,7 +10,13 @@ from pandas.testing import assert_frame_equal from feast.client import Client -from feast.data_source import BigQuerySource, FileSource, KafkaSource, ParquetFormat, ProtoFormat +from feast.data_source import ( + BigQuerySource, + FileSource, + KafkaSource, + ParquetFormat, + ProtoFormat, +) from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable From 6b51d354f941d1b822eebbef2ea44a586014e5df Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 17:25:41 +0800 Subject: [PATCH 12/24] E2E: Fix typo Signed-off-by: Zhu Zhanyan --- tests/e2e/test-register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index 7d2c8a7c6c7..340bd74a18f 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -19,7 +19,7 @@ ) from feast.entity import Entity from feast.feature import Feature -from feast.feature_table import FeatureTable +from feast.feature_table import FeatureTable) from feast.value_type import ValueType from feast.wait import wait_retry_backoff @@ -150,7 +150,7 @@ def alltypes_entity(): @pytest.fixture def alltypes_featuretable(): batch_source = FileSource( - file_format="parquet", + file_format=ParquetFormat(), file_url="file://feast/*", event_timestamp_column="ts_col", created_timestamp_column="timestamp", From 0e3092d53fbeeb3e053fb6ee4ea26642d57a0e24 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 18:24:54 +0800 Subject: [PATCH 13/24] Serving: Fix IT TestUtils not producing Protobuf specs inline with the current API Signed-off-by: Zhu Zhanyan --- .../java/feast/common/it/DataGenerator.java | 2 +- .../java/feast/core/model/DataSourceTest.java | 2 +- .../feast/core/service/SpecServiceIT.java | 24 ++++----- .../validators/DataSourceValidatorTest.java | 2 +- .../test/java/feast/serving/it/TestUtils.java | 53 ++----------------- 5 files changed, 19 insertions(+), 64 deletions(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 6f7b37251b2..fe8ebf56e8c 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -271,7 +271,7 @@ public static FeatureTableSpec createFeatureTableSpec( } public static DataSource createFileDataSourceSpec( - String fileURL, String fileFormat, String timestampColumn, String datePartitionColumn) { + String fileURL, String timestampColumn, String datePartitionColumn) { return DataSource.newBuilder() .setType(DataSource.SourceType.BATCH_FILE) .setFileOptions( diff --git a/core/src/test/java/feast/core/model/DataSourceTest.java b/core/src/test/java/feast/core/model/DataSourceTest.java index 20445ecb441..2dcbaa93cd3 100644 --- a/core/src/test/java/feast/core/model/DataSourceTest.java +++ b/core/src/test/java/feast/core/model/DataSourceTest.java @@ -52,7 +52,7 @@ public void shouldFromProtoBeReversableWithToProto() { private List getTestSpecs() { return List.of( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "parquet", "ts_col", ""), + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), DataGenerator.createKafkaDataSourceSpec("localhost:9092", "topic", "class.path", "ts_col"), DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), DataGenerator.createKinesisDataSourceSpec("ap-nowhere1", "stream", "class.path", "ts_col")); diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index f4713c8ab7c..31b73466662 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -101,7 +101,7 @@ public void initState() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build()); apiClient.simpleApplyEntity( "project1", @@ -954,7 +954,7 @@ public void shouldReturnFeatureTableIfExists() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build(); FeatureTableProto.FeatureTable featureTable = apiClient.simpleGetFeatureTable("default", "featuretable1"); @@ -1067,7 +1067,7 @@ private FeatureTableSpec getTestSpec() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .setStreamSource( DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col")) @@ -1099,7 +1099,7 @@ public void shouldUpdateExistingTableWithValidSpec() { .toBuilder() .setStreamSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .setBatchSource( DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col")) @@ -1159,7 +1159,7 @@ public void shouldErrorIfEntityChangeOnUpdate() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build(); StatusRuntimeException exc = @@ -1191,7 +1191,7 @@ public void shouldErrorIfFeatureValueTypeChangeOnUpdate() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build(); StatusRuntimeException exc = @@ -1250,7 +1250,7 @@ public void shouldErrorOnReservedNames() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); // Reserved name used in as entity name @@ -1268,7 +1268,7 @@ public void shouldErrorOnReservedNames() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); } @@ -1289,7 +1289,7 @@ public void shouldErrorOnInvalidName() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); // Invalid feature name @@ -1307,7 +1307,7 @@ public void shouldErrorOnInvalidName() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); } @@ -1327,7 +1327,7 @@ public void shouldErrorOnNotFoundEntityName() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); } @@ -1350,7 +1350,7 @@ public void shouldErrorOnArchivedProject() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "parquet", "ts_col", "")) + "file:///path/to/file", "ts_col", "")) .build())); } } diff --git a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java index d3400379b33..cccaed88c24 100644 --- a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java +++ b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java @@ -103,7 +103,7 @@ public void shouldErrorOnUnsupportedDataFormats() { private Map getTestSpecsMap() { return Map.of( BATCH_FILE, - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "parquet", "ts_col", ""), + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), BATCH_BIGQUERY, DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col"), diff --git a/serving/src/test/java/feast/serving/it/TestUtils.java b/serving/src/test/java/feast/serving/it/TestUtils.java index 4903a8c4d0f..ac2191a62ae 100644 --- a/serving/src/test/java/feast/serving/it/TestUtils.java +++ b/serving/src/test/java/feast/serving/it/TestUtils.java @@ -21,9 +21,11 @@ import com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; import feast.common.auth.credentials.OAuthCredentials; +import feast.common.it.DataGenerator; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; import feast.proto.core.DataSourceProto.DataSource; +import feast.proto.core.DataSourceProto.DataFormat.ParquetFormat; import feast.proto.core.EntityProto.Entity; import feast.proto.core.EntityProto.EntitySpecV2; import feast.proto.core.FeatureProto.FeatureSpecV2; @@ -64,44 +66,6 @@ public static CoreSimpleAPIClient getApiClientForCore(int feastCorePort) { return new CoreSimpleAPIClient(coreService); } - public static FeatureTableSpec createFeatureTableSpec( - String name, - List entities, - Map features, - int maxAgeSecs, - Map labels) { - return FeatureTableSpec.newBuilder() - .setName(name) - .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSecs).build()) - .addAllEntities(entities) - .addAllFeatures( - features.entrySet().stream() - .map( - entry -> - FeatureSpecV2.newBuilder() - .setName(entry.getKey()) - .setValueType(entry.getValue()) - .putAllLabels(labels) - .build()) - .collect(Collectors.toList())) - .putAllLabels(labels) - .build(); - } - - public static DataSource createFileDataSourceSpec( - String fileURL, String fileFormat, String timestampColumn, String datePartitionColumn) { - return DataSource.newBuilder() - .setType(DataSource.SourceType.BATCH_FILE) - .setFileOptions( - DataSource.FileOptions.newBuilder() - .setFileFormat(fileFormat) - .setFileUrl(fileURL) - .build()) - .setEventTimestampColumn(timestampColumn) - .setDatePartitionColumn(datePartitionColumn) - .build(); - } - public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( String projectName, List featureReferences, @@ -121,19 +85,10 @@ public static void applyFeatureTable( ImmutableMap features, int maxAgeSecs) { FeatureTableSpec expectedFeatureTableSpec = - createFeatureTableSpec( - featureTableName, - entities, - new HashMap<>() { - { - putAll(features); - } - }, - maxAgeSecs, - ImmutableMap.of("feat_key2", "feat_value2")) + DataGenerator.createFeatureTableSpec(featureTableName, entities, features, maxAgeSecs, Map.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( - createFileDataSourceSpec("file:///path/to/file", "parquet", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) .build(); secureApiClient.simpleApplyFeatureTable(expectedFeatureTableSpec); FeatureTable actualFeatureTable = From afdfc8ae6c798d1e37b256949bf55d5bcb07d2c4 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 18:33:15 +0800 Subject: [PATCH 14/24] Fix java lint Signed-off-by: Zhu Zhanyan --- .../java/feast/core/service/SpecServiceIT.java | 18 ++++++------------ .../validators/DataSourceValidatorTest.java | 3 +-- .../test/java/feast/serving/it/TestUtils.java | 12 ++++++------ 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 31b73466662..35535d07157 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -100,8 +100,7 @@ public void initState() { ImmutableMap.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .build()); apiClient.simpleApplyEntity( "project1", @@ -953,8 +952,7 @@ public void shouldReturnFeatureTableIfExists() { ImmutableMap.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .build(); FeatureTableProto.FeatureTable featureTable = apiClient.simpleGetFeatureTable("default", "featuretable1"); @@ -1066,8 +1064,7 @@ private FeatureTableSpec getTestSpec() { Map.of()) .toBuilder() .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .setStreamSource( DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col")) @@ -1098,8 +1095,7 @@ public void shouldUpdateExistingTableWithValidSpec() { Map.of("test", "labels")) .toBuilder() .setStreamSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .setBatchSource( DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col")) @@ -1158,8 +1154,7 @@ public void shouldErrorIfEntityChangeOnUpdate() { ImmutableMap.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .build(); StatusRuntimeException exc = @@ -1190,8 +1185,7 @@ public void shouldErrorIfFeatureValueTypeChangeOnUpdate() { ImmutableMap.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) .build(); StatusRuntimeException exc = diff --git a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java index cccaed88c24..3bbd6e86d7e 100644 --- a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java +++ b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java @@ -102,8 +102,7 @@ public void shouldErrorOnUnsupportedDataFormats() { private Map getTestSpecsMap() { return Map.of( - BATCH_FILE, - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), + BATCH_FILE, DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), BATCH_BIGQUERY, DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col"), diff --git a/serving/src/test/java/feast/serving/it/TestUtils.java b/serving/src/test/java/feast/serving/it/TestUtils.java index ac2191a62ae..0c72df1ea86 100644 --- a/serving/src/test/java/feast/serving/it/TestUtils.java +++ b/serving/src/test/java/feast/serving/it/TestUtils.java @@ -19,16 +19,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.ImmutableMap; -import com.google.protobuf.Duration; import feast.common.auth.credentials.OAuthCredentials; import feast.common.it.DataGenerator; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.DataSourceProto.DataSource; -import feast.proto.core.DataSourceProto.DataFormat.ParquetFormat; import feast.proto.core.EntityProto.Entity; import feast.proto.core.EntityProto.EntitySpecV2; -import feast.proto.core.FeatureProto.FeatureSpecV2; import feast.proto.core.FeatureTableProto.FeatureTable; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; @@ -39,7 +35,6 @@ import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; import java.util.*; -import java.util.stream.Collectors; public class TestUtils { @@ -85,7 +80,12 @@ public static void applyFeatureTable( ImmutableMap features, int maxAgeSecs) { FeatureTableSpec expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec(featureTableName, entities, features, maxAgeSecs, Map.of("feat_key2", "feat_value2")) + DataGenerator.createFeatureTableSpec( + featureTableName, + entities, + features, + maxAgeSecs, + Map.of("feat_key2", "feat_value2")) .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) From 919856ae579a7fa78b0af18cdb9942622943616f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 14 Oct 2020 18:44:17 +0800 Subject: [PATCH 15/24] E2E: Fix another typo Signed-off-by: Zhu Zhanyan --- tests/e2e/test-register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index 340bd74a18f..4eb67ad949f 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -19,7 +19,7 @@ ) from feast.entity import Entity from feast.feature import Feature -from feast.feature_table import FeatureTable) +from feast.feature_table import FeatureTable from feast.value_type import ValueType from feast.wait import wait_retry_backoff From 92c8319ffc3ca051421d4f2af2d3d8092fff4232 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 16 Oct 2020 12:53:03 +0800 Subject: [PATCH 16/24] Proto: Split DataFormat message into StreamFormat and FileFormat messages Signed-off-by: Zhu Zhanyan --- go.mod | 4 +- go.sum | 6 +- protos/feast/core/DataSource.proto | 33 +- sdk/go/protos/feast/core/CoreService.pb.go | 2 +- sdk/go/protos/feast/core/DataSource.pb.go | 663 ++++++++++++++---- sdk/go/protos/feast/core/Entity.pb.go | 2 +- sdk/go/protos/feast/core/Feature.pb.go | 2 +- sdk/go/protos/feast/core/FeatureSet.pb.go | 2 +- .../feast/core/FeatureSetReference.pb.go | 2 +- sdk/go/protos/feast/core/FeatureTable.pb.go | 2 +- sdk/go/protos/feast/core/IngestionJob.pb.go | 2 +- sdk/go/protos/feast/core/Runner.pb.go | 2 +- sdk/go/protos/feast/core/Source.pb.go | 2 +- sdk/go/protos/feast/core/Store.pb.go | 2 +- .../protos/feast/serving/ServingService.pb.go | 2 +- sdk/go/protos/feast/storage/Redis.pb.go | 2 +- sdk/go/protos/feast/types/FeatureRow.pb.go | 2 +- .../feast/types/FeatureRowExtended.pb.go | 2 +- sdk/go/protos/feast/types/Field.pb.go | 2 +- sdk/go/protos/feast/types/Value.pb.go | 2 +- .../tensorflow_metadata/proto/v0/path.pb.go | 2 +- .../tensorflow_metadata/proto/v0/schema.pb.go | 2 +- .../proto/v0/statistics.pb.go | 2 +- 23 files changed, 572 insertions(+), 172 deletions(-) diff --git a/go.mod b/go.mod index 8b36dfe6788..c7c3cbd9060 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/mock v1.2.0 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.4.3 github.com/google/go-cmp v0.5.0 github.com/huandu/xstrings v1.2.0 // indirect github.com/lyft/protoc-gen-validate v0.1.0 // indirect @@ -25,7 +25,7 @@ require ( golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 // indirect + golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.25.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 4e26be46a3c..046c0ae5725 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -494,8 +496,8 @@ golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 h1:BANdcOVw3KTuUiyfDp7wrzC golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201013053347-2db1cd791039 h1:kLBxO4OPBgPwjg8Vvu+/0DCHIfDwYIGNFcD66NU9kpo= golang.org/x/tools v0.0.0-20201013053347-2db1cd791039/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 h1:2ntEwh02rqo2jSsrYmp4yKHHjh0CbXP3ZtSUetSB+q8= -golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4 h1:rQWkJiVIyJ3PgiSHL+RXc8xbrK8duU6jG5eeZ9G7nk8= +golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 6148c15bcb4..4f5fbf27476 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -22,8 +22,18 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; option java_outer_classname = "DataSourceProto"; option java_package = "feast.proto.core"; -// Defines the data format and format specific options -message DataFormat { +// Defines the file format encoding the features/entity data in files +message FileFormat { + // Defines options for the Parquet data format + message ParquetFormat {} + + oneof format { + ParquetFormat parquet_format = 1; + } +} + +// Defines the data format encoding features/entity data in data streams +message StreamFormat { // Defines options for the protobuf data format message ProtoFormat { // Classpath to the generated Java Protobuf class that can be used to decode @@ -38,14 +48,10 @@ message DataFormat { string schema_json = 1; } - // Defines options for the Parquet data format - message ParquetFormat {} - // Specifies the data format and format specific options oneof format { - ProtoFormat proto_format = 1; - AvroFormat avro_format = 2; - ParquetFormat parquet_format = 3; + AvroFormat avro_format = 1; + ProtoFormat proto_format = 2; } } @@ -77,9 +83,7 @@ message DataSource { // Defines options for DataSource that sources features from a file message FileOptions { - // Defines the file format encoding the features/entity data - // File Data Sources support Avro and Parquet as data formats. - DataFormat file_format = 1; + FileFormat file_format = 1; // Target URL of file to retrieve and source features from. // s3://path/to/file for AWS S3 storage @@ -104,9 +108,8 @@ message DataSource { // Kafka topic to collect feature data from. string topic = 2; - // Defines the data format encoding feature/entity data in Kafka messages. - // Kafka Data Sources support Avro and Proto as data formats. - DataFormat message_format = 3; + // Defines the stream data format encoding feature/entity data in Kafka messages. + StreamFormat message_format = 3; } // Defines options for DataSource that sources features from Kinesis records. @@ -121,7 +124,7 @@ message DataSource { // Defines the data format encoding the feature/entity data in Kinesis records. // Kinesis Data Sources support Avro and Proto as data formats. - DataFormat record_format = 3; + StreamFormat record_format = 3; } // DataSource options. diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 53f82b9a6a4..c54a5bc3151 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/CoreService.proto package core diff --git a/sdk/go/protos/feast/core/DataSource.pb.go b/sdk/go/protos/feast/core/DataSource.pb.go index a70176bccb9..b082e8bde6e 100644 --- a/sdk/go/protos/feast/core/DataSource.pb.go +++ b/sdk/go/protos/feast/core/DataSource.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/DataSource.proto package core @@ -94,9 +94,159 @@ func (x DataSource_SourceType) Number() protoreflect.EnumNumber { // Deprecated: Use DataSource_SourceType.Descriptor instead. func (DataSource_SourceType) EnumDescriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 0} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 0} +} + +// Defines the file format encoding the features/entity data in files +type FileFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Format: + // *FileFormat_ParquetFormat_ + Format isFileFormat_Format `protobuf_oneof:"format"` +} + +func (x *FileFormat) Reset() { + *x = FileFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileFormat) ProtoMessage() {} + +func (x *FileFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileFormat.ProtoReflect.Descriptor instead. +func (*FileFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0} } +func (m *FileFormat) GetFormat() isFileFormat_Format { + if m != nil { + return m.Format + } + return nil +} + +func (x *FileFormat) GetParquetFormat() *FileFormat_ParquetFormat { + if x, ok := x.GetFormat().(*FileFormat_ParquetFormat_); ok { + return x.ParquetFormat + } + return nil +} + +type isFileFormat_Format interface { + isFileFormat_Format() +} + +type FileFormat_ParquetFormat_ struct { + ParquetFormat *FileFormat_ParquetFormat `protobuf:"bytes,1,opt,name=parquet_format,json=parquetFormat,proto3,oneof"` +} + +func (*FileFormat_ParquetFormat_) isFileFormat_Format() {} + +// Defines the data format encoding features/entity data in data streams +type StreamFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Specifies the data format and format specific options + // + // Types that are assignable to Format: + // *StreamFormat_AvroFormat_ + // *StreamFormat_ProtoFormat_ + Format isStreamFormat_Format `protobuf_oneof:"format"` +} + +func (x *StreamFormat) Reset() { + *x = StreamFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat) ProtoMessage() {} + +func (x *StreamFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1} +} + +func (m *StreamFormat) GetFormat() isStreamFormat_Format { + if m != nil { + return m.Format + } + return nil +} + +func (x *StreamFormat) GetAvroFormat() *StreamFormat_AvroFormat { + if x, ok := x.GetFormat().(*StreamFormat_AvroFormat_); ok { + return x.AvroFormat + } + return nil +} + +func (x *StreamFormat) GetProtoFormat() *StreamFormat_ProtoFormat { + if x, ok := x.GetFormat().(*StreamFormat_ProtoFormat_); ok { + return x.ProtoFormat + } + return nil +} + +type isStreamFormat_Format interface { + isStreamFormat_Format() +} + +type StreamFormat_AvroFormat_ struct { + AvroFormat *StreamFormat_AvroFormat `protobuf:"bytes,1,opt,name=avro_format,json=avroFormat,proto3,oneof"` +} + +type StreamFormat_ProtoFormat_ struct { + ProtoFormat *StreamFormat_ProtoFormat `protobuf:"bytes,2,opt,name=proto_format,json=protoFormat,proto3,oneof"` +} + +func (*StreamFormat_AvroFormat_) isStreamFormat_Format() {} + +func (*StreamFormat_ProtoFormat_) isStreamFormat_Format() {} + // Defines a Data Source that can be used source Feature data type DataSource struct { state protoimpl.MessageState @@ -127,7 +277,7 @@ type DataSource struct { func (x *DataSource) Reset() { *x = DataSource{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[0] + mi := &file_feast_core_DataSource_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -140,7 +290,7 @@ func (x *DataSource) String() string { func (*DataSource) ProtoMessage() {} func (x *DataSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[0] + mi := &file_feast_core_DataSource_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -153,7 +303,7 @@ func (x *DataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource.ProtoReflect.Descriptor instead. func (*DataSource) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2} } func (x *DataSource) GetType() DataSource_SourceType { @@ -254,14 +404,152 @@ func (*DataSource_KafkaOptions_) isDataSource_Options() {} func (*DataSource_KinesisOptions_) isDataSource_Options() {} +// Defines options for the Parquet data format +type FileFormat_ParquetFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *FileFormat_ParquetFormat) Reset() { + *x = FileFormat_ParquetFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileFormat_ParquetFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileFormat_ParquetFormat) ProtoMessage() {} + +func (x *FileFormat_ParquetFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileFormat_ParquetFormat.ProtoReflect.Descriptor instead. +func (*FileFormat_ParquetFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 0} +} + +// Defines options for the protobuf data format +type StreamFormat_ProtoFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Classpath to the generated Java Protobuf class that can be used to decode + // Feature data from the obtained stream message + ClassPath string `protobuf:"bytes,1,opt,name=class_path,json=classPath,proto3" json:"class_path,omitempty"` +} + +func (x *StreamFormat_ProtoFormat) Reset() { + *x = StreamFormat_ProtoFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat_ProtoFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat_ProtoFormat) ProtoMessage() {} + +func (x *StreamFormat_ProtoFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat_ProtoFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat_ProtoFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *StreamFormat_ProtoFormat) GetClassPath() string { + if x != nil { + return x.ClassPath + } + return "" +} + +// Defines options for the avro data format +type StreamFormat_AvroFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional if used in a File DataSource as schema is embedded in avro file. + // Specifies the schema of the Avro message as JSON string. + SchemaJson string `protobuf:"bytes,1,opt,name=schema_json,json=schemaJson,proto3" json:"schema_json,omitempty"` +} + +func (x *StreamFormat_AvroFormat) Reset() { + *x = StreamFormat_AvroFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat_AvroFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat_AvroFormat) ProtoMessage() {} + +func (x *StreamFormat_AvroFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat_AvroFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat_AvroFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *StreamFormat_AvroFormat) GetSchemaJson() string { + if x != nil { + return x.SchemaJson + } + return "" +} + // Defines options for DataSource that sources features from a file type DataSource_FileOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // File Format of the file containing the features - FileFormat string `protobuf:"bytes,1,opt,name=file_format,json=fileFormat,proto3" json:"file_format,omitempty"` + FileFormat *FileFormat `protobuf:"bytes,1,opt,name=file_format,json=fileFormat,proto3" json:"file_format,omitempty"` // Target URL of file to retrieve and source features from. // s3://path/to/file for AWS S3 storage // gs://path/to/file for GCP GCS storage @@ -272,7 +560,7 @@ type DataSource_FileOptions struct { func (x *DataSource_FileOptions) Reset() { *x = DataSource_FileOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[2] + mi := &file_feast_core_DataSource_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -285,7 +573,7 @@ func (x *DataSource_FileOptions) String() string { func (*DataSource_FileOptions) ProtoMessage() {} func (x *DataSource_FileOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[2] + mi := &file_feast_core_DataSource_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -298,14 +586,14 @@ func (x *DataSource_FileOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_FileOptions.ProtoReflect.Descriptor instead. func (*DataSource_FileOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 1} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 1} } -func (x *DataSource_FileOptions) GetFileFormat() string { +func (x *DataSource_FileOptions) GetFileFormat() *FileFormat { if x != nil { return x.FileFormat } - return "" + return nil } func (x *DataSource_FileOptions) GetFileUrl() string { @@ -328,7 +616,7 @@ type DataSource_BigQueryOptions struct { func (x *DataSource_BigQueryOptions) Reset() { *x = DataSource_BigQueryOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[3] + mi := &file_feast_core_DataSource_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -341,7 +629,7 @@ func (x *DataSource_BigQueryOptions) String() string { func (*DataSource_BigQueryOptions) ProtoMessage() {} func (x *DataSource_BigQueryOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[3] + mi := &file_feast_core_DataSource_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -354,7 +642,7 @@ func (x *DataSource_BigQueryOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_BigQueryOptions.ProtoReflect.Descriptor instead. func (*DataSource_BigQueryOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 2} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 2} } func (x *DataSource_BigQueryOptions) GetTableRef() string { @@ -376,15 +664,14 @@ type DataSource_KafkaOptions struct { BootstrapServers string `protobuf:"bytes,1,opt,name=bootstrap_servers,json=bootstrapServers,proto3" json:"bootstrap_servers,omitempty"` // Kafka topic to collect feature data from. Topic string `protobuf:"bytes,2,opt,name=topic,proto3" json:"topic,omitempty"` - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained Kafka message - ClassPath string `protobuf:"bytes,3,opt,name=class_path,json=classPath,proto3" json:"class_path,omitempty"` + // Defines the stream data format encoding feature/entity data in Kafka messages. + MessageFormat *StreamFormat `protobuf:"bytes,3,opt,name=message_format,json=messageFormat,proto3" json:"message_format,omitempty"` } func (x *DataSource_KafkaOptions) Reset() { *x = DataSource_KafkaOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[4] + mi := &file_feast_core_DataSource_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -397,7 +684,7 @@ func (x *DataSource_KafkaOptions) String() string { func (*DataSource_KafkaOptions) ProtoMessage() {} func (x *DataSource_KafkaOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[4] + mi := &file_feast_core_DataSource_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -410,7 +697,7 @@ func (x *DataSource_KafkaOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_KafkaOptions.ProtoReflect.Descriptor instead. func (*DataSource_KafkaOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 3} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 3} } func (x *DataSource_KafkaOptions) GetBootstrapServers() string { @@ -427,11 +714,11 @@ func (x *DataSource_KafkaOptions) GetTopic() string { return "" } -func (x *DataSource_KafkaOptions) GetClassPath() string { +func (x *DataSource_KafkaOptions) GetMessageFormat() *StreamFormat { if x != nil { - return x.ClassPath + return x.MessageFormat } - return "" + return nil } // Defines options for DataSource that sources features from Kinesis records. @@ -446,15 +733,15 @@ type DataSource_KinesisOptions struct { Region string `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"` // Name of the Kinesis stream to obtain feature data from. StreamName string `protobuf:"bytes,2,opt,name=stream_name,json=streamName,proto3" json:"stream_name,omitempty"` - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained Kinesis record - ClassPath string `protobuf:"bytes,3,opt,name=class_path,json=classPath,proto3" json:"class_path,omitempty"` + // Defines the data format encoding the feature/entity data in Kinesis records. + // Kinesis Data Sources support Avro and Proto as data formats. + RecordFormat *StreamFormat `protobuf:"bytes,3,opt,name=record_format,json=recordFormat,proto3" json:"record_format,omitempty"` } func (x *DataSource_KinesisOptions) Reset() { *x = DataSource_KinesisOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[5] + mi := &file_feast_core_DataSource_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -467,7 +754,7 @@ func (x *DataSource_KinesisOptions) String() string { func (*DataSource_KinesisOptions) ProtoMessage() {} func (x *DataSource_KinesisOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[5] + mi := &file_feast_core_DataSource_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -480,7 +767,7 @@ func (x *DataSource_KinesisOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_KinesisOptions.ProtoReflect.Descriptor instead. func (*DataSource_KinesisOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 4} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 4} } func (x *DataSource_KinesisOptions) GetRegion() string { @@ -497,11 +784,11 @@ func (x *DataSource_KinesisOptions) GetStreamName() string { return "" } -func (x *DataSource_KinesisOptions) GetClassPath() string { +func (x *DataSource_KinesisOptions) GetRecordFormat() *StreamFormat { if x != nil { - return x.ClassPath + return x.RecordFormat } - return "" + return nil } var File_feast_core_DataSource_proto protoreflect.FileDescriptor @@ -509,85 +796,115 @@ var File_feast_core_DataSource_proto protoreflect.FileDescriptor var file_feast_core_DataSource_proto_rawDesc = []byte{ 0x0a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0xfa, 0x08, 0x0a, 0x0a, 0x44, 0x61, - 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, - 0x4d, 0x0a, 0x0d, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x34, - 0x0a, 0x16, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x63, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x63, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x12, 0x47, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x76, 0x0a, 0x0a, 0x46, 0x69, 0x6c, + 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x4d, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x71, 0x75, + 0x65, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, + 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, + 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x0f, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x22, 0x88, 0x02, 0x0a, 0x0c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x12, 0x46, 0x0a, 0x0b, 0x61, 0x76, 0x72, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x2e, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0a, + 0x61, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x49, 0x0a, 0x0c, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x2c, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x46, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x50, + 0x61, 0x74, 0x68, 0x1a, 0x2d, 0x0a, 0x0a, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4a, 0x73, + 0x6f, 0x6e, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd6, 0x09, 0x0a, + 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x0d, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x12, 0x34, 0x0a, 0x16, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x14, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, 0x74, 0x65, 0x5f, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x47, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, + 0x00, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, + 0x0a, 0x10, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0b, - 0x66, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, 0x10, 0x62, - 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x42, 0x69, - 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, - 0x0f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x4a, 0x0a, 0x0d, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, - 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0c, - 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x0f, - 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x69, - 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0e, - 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3f, - 0x0a, 0x11, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x49, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, - 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, - 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x1a, 0x2e, 0x0a, 0x0f, 0x42, 0x69, - 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x0a, - 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x66, 0x1a, 0x70, 0x0a, 0x0c, 0x4b, 0x61, - 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, - 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x1d, 0x0a, - 0x0a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x50, 0x61, 0x74, 0x68, 0x1a, 0x68, 0x0a, 0x0e, - 0x4b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, - 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x61, 0x73, 0x73, - 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, - 0x73, 0x73, 0x50, 0x61, 0x74, 0x68, 0x22, 0x63, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, - 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, - 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, - 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, - 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, - 0x4d, 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, 0x49, 0x53, 0x10, 0x04, 0x42, 0x09, 0x0a, 0x07, 0x6f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, - 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, - 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x48, 0x00, 0x52, 0x0f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, + 0x00, 0x52, 0x0c, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x50, 0x0a, 0x0f, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x4b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, + 0x00, 0x52, 0x0e, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x61, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, + 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, + 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, + 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x1a, 0x2e, 0x0a, 0x0f, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x52, 0x65, 0x66, 0x1a, 0x92, 0x01, 0x0a, 0x0c, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, + 0x72, 0x61, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x3f, 0x0a, 0x0e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0d, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x88, 0x01, 0x0a, 0x0e, 0x4b, + 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0x63, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, + 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, + 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, + 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, + 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, + 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, 0x49, 0x53, 0x10, 0x04, 0x42, 0x09, 0x0a, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -603,28 +920,39 @@ func file_feast_core_DataSource_proto_rawDescGZIP() []byte { } var file_feast_core_DataSource_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_feast_core_DataSource_proto_goTypes = []interface{}{ (DataSource_SourceType)(0), // 0: feast.core.DataSource.SourceType - (*DataSource)(nil), // 1: feast.core.DataSource - nil, // 2: feast.core.DataSource.FieldMappingEntry - (*DataSource_FileOptions)(nil), // 3: feast.core.DataSource.FileOptions - (*DataSource_BigQueryOptions)(nil), // 4: feast.core.DataSource.BigQueryOptions - (*DataSource_KafkaOptions)(nil), // 5: feast.core.DataSource.KafkaOptions - (*DataSource_KinesisOptions)(nil), // 6: feast.core.DataSource.KinesisOptions + (*FileFormat)(nil), // 1: feast.core.FileFormat + (*StreamFormat)(nil), // 2: feast.core.StreamFormat + (*DataSource)(nil), // 3: feast.core.DataSource + (*FileFormat_ParquetFormat)(nil), // 4: feast.core.FileFormat.ParquetFormat + (*StreamFormat_ProtoFormat)(nil), // 5: feast.core.StreamFormat.ProtoFormat + (*StreamFormat_AvroFormat)(nil), // 6: feast.core.StreamFormat.AvroFormat + nil, // 7: feast.core.DataSource.FieldMappingEntry + (*DataSource_FileOptions)(nil), // 8: feast.core.DataSource.FileOptions + (*DataSource_BigQueryOptions)(nil), // 9: feast.core.DataSource.BigQueryOptions + (*DataSource_KafkaOptions)(nil), // 10: feast.core.DataSource.KafkaOptions + (*DataSource_KinesisOptions)(nil), // 11: feast.core.DataSource.KinesisOptions } var file_feast_core_DataSource_proto_depIdxs = []int32{ - 0, // 0: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType - 2, // 1: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry - 3, // 2: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions - 4, // 3: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions - 5, // 4: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions - 6, // 5: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 4, // 0: feast.core.FileFormat.parquet_format:type_name -> feast.core.FileFormat.ParquetFormat + 6, // 1: feast.core.StreamFormat.avro_format:type_name -> feast.core.StreamFormat.AvroFormat + 5, // 2: feast.core.StreamFormat.proto_format:type_name -> feast.core.StreamFormat.ProtoFormat + 0, // 3: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType + 7, // 4: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry + 8, // 5: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions + 9, // 6: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions + 10, // 7: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions + 11, // 8: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions + 1, // 9: feast.core.DataSource.FileOptions.file_format:type_name -> feast.core.FileFormat + 2, // 10: feast.core.DataSource.KafkaOptions.message_format:type_name -> feast.core.StreamFormat + 2, // 11: feast.core.DataSource.KinesisOptions.record_format:type_name -> feast.core.StreamFormat + 12, // [12:12] is the sub-list for method output_type + 12, // [12:12] is the sub-list for method input_type + 12, // [12:12] is the sub-list for extension type_name + 12, // [12:12] is the sub-list for extension extendee + 0, // [0:12] is the sub-list for field type_name } func init() { file_feast_core_DataSource_proto_init() } @@ -634,7 +962,19 @@ func file_feast_core_DataSource_proto_init() { } if !protoimpl.UnsafeEnabled { file_feast_core_DataSource_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DataSource); i { + switch v := v.(*FileFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamFormat); i { case 0: return &v.state case 1: @@ -646,7 +986,7 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DataSource_FileOptions); i { + switch v := v.(*DataSource); i { case 0: return &v.state case 1: @@ -658,7 +998,7 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DataSource_BigQueryOptions); i { + switch v := v.(*FileFormat_ParquetFormat); i { case 0: return &v.state case 1: @@ -670,7 +1010,7 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DataSource_KafkaOptions); i { + switch v := v.(*StreamFormat_ProtoFormat); i { case 0: return &v.state case 1: @@ -682,6 +1022,54 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamFormat_AvroFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_FileOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_BigQueryOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_KafkaOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource_KinesisOptions); i { case 0: return &v.state @@ -695,6 +1083,13 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*FileFormat_ParquetFormat_)(nil), + } + file_feast_core_DataSource_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*StreamFormat_AvroFormat_)(nil), + (*StreamFormat_ProtoFormat_)(nil), + } + file_feast_core_DataSource_proto_msgTypes[2].OneofWrappers = []interface{}{ (*DataSource_FileOptions_)(nil), (*DataSource_BigqueryOptions)(nil), (*DataSource_KafkaOptions_)(nil), @@ -706,7 +1101,7 @@ func file_feast_core_DataSource_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_DataSource_proto_rawDesc, NumEnums: 1, - NumMessages: 6, + NumMessages: 11, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/core/Entity.pb.go b/sdk/go/protos/feast/core/Entity.pb.go index 0aed9133257..f5a2c0d06af 100644 --- a/sdk/go/protos/feast/core/Entity.pb.go +++ b/sdk/go/protos/feast/core/Entity.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/Entity.proto package core diff --git a/sdk/go/protos/feast/core/Feature.pb.go b/sdk/go/protos/feast/core/Feature.pb.go index 1ad93ef8a1c..9c2690e0f9f 100644 --- a/sdk/go/protos/feast/core/Feature.pb.go +++ b/sdk/go/protos/feast/core/Feature.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/Feature.proto package core diff --git a/sdk/go/protos/feast/core/FeatureSet.pb.go b/sdk/go/protos/feast/core/FeatureSet.pb.go index d13041a8681..520c4881be5 100644 --- a/sdk/go/protos/feast/core/FeatureSet.pb.go +++ b/sdk/go/protos/feast/core/FeatureSet.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/FeatureSet.proto package core diff --git a/sdk/go/protos/feast/core/FeatureSetReference.pb.go b/sdk/go/protos/feast/core/FeatureSetReference.pb.go index d82e94b6a1c..ca640aa0fb3 100644 --- a/sdk/go/protos/feast/core/FeatureSetReference.pb.go +++ b/sdk/go/protos/feast/core/FeatureSetReference.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/FeatureSetReference.proto package core diff --git a/sdk/go/protos/feast/core/FeatureTable.pb.go b/sdk/go/protos/feast/core/FeatureTable.pb.go index 290477931d0..48aef52950a 100644 --- a/sdk/go/protos/feast/core/FeatureTable.pb.go +++ b/sdk/go/protos/feast/core/FeatureTable.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/FeatureTable.proto package core diff --git a/sdk/go/protos/feast/core/IngestionJob.pb.go b/sdk/go/protos/feast/core/IngestionJob.pb.go index 047a9cea0e6..a716bdf0893 100644 --- a/sdk/go/protos/feast/core/IngestionJob.pb.go +++ b/sdk/go/protos/feast/core/IngestionJob.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/IngestionJob.proto package core diff --git a/sdk/go/protos/feast/core/Runner.pb.go b/sdk/go/protos/feast/core/Runner.pb.go index 105878c6d60..763695f6c8e 100644 --- a/sdk/go/protos/feast/core/Runner.pb.go +++ b/sdk/go/protos/feast/core/Runner.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/Runner.proto package core diff --git a/sdk/go/protos/feast/core/Source.pb.go b/sdk/go/protos/feast/core/Source.pb.go index a5b3de95647..af7f9783bbf 100644 --- a/sdk/go/protos/feast/core/Source.pb.go +++ b/sdk/go/protos/feast/core/Source.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/Source.proto package core diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index c037fe84b53..b5ffae9326f 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/core/Store.proto package core diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index dc99f285104..f77930562ed 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/serving/ServingService.proto package serving diff --git a/sdk/go/protos/feast/storage/Redis.pb.go b/sdk/go/protos/feast/storage/Redis.pb.go index d169c8bf321..7b3a3f6a774 100644 --- a/sdk/go/protos/feast/storage/Redis.pb.go +++ b/sdk/go/protos/feast/storage/Redis.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/storage/Redis.proto package storage diff --git a/sdk/go/protos/feast/types/FeatureRow.pb.go b/sdk/go/protos/feast/types/FeatureRow.pb.go index 0c61f25f3d0..f6fe6bfa424 100644 --- a/sdk/go/protos/feast/types/FeatureRow.pb.go +++ b/sdk/go/protos/feast/types/FeatureRow.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/types/FeatureRow.proto package types diff --git a/sdk/go/protos/feast/types/FeatureRowExtended.pb.go b/sdk/go/protos/feast/types/FeatureRowExtended.pb.go index d1e02b1e0ae..a01f0b0417a 100644 --- a/sdk/go/protos/feast/types/FeatureRowExtended.pb.go +++ b/sdk/go/protos/feast/types/FeatureRowExtended.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/types/FeatureRowExtended.proto package types diff --git a/sdk/go/protos/feast/types/Field.pb.go b/sdk/go/protos/feast/types/Field.pb.go index 9dad77cdb91..f2562b72d95 100644 --- a/sdk/go/protos/feast/types/Field.pb.go +++ b/sdk/go/protos/feast/types/Field.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/types/Field.proto package types diff --git a/sdk/go/protos/feast/types/Value.pb.go b/sdk/go/protos/feast/types/Value.pb.go index 3b194356331..3625cef1a53 100644 --- a/sdk/go/protos/feast/types/Value.pb.go +++ b/sdk/go/protos/feast/types/Value.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: feast/types/Value.proto package types diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index 1daa7687f94..a1e5137c727 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: tensorflow_metadata/proto/v0/path.proto package v0 diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index 940779a1917..25bf40bc7fe 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: tensorflow_metadata/proto/v0/schema.proto package v0 diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go index fbf6247a1d3..6a102a28afa 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go @@ -20,7 +20,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.10.0 // source: tensorflow_metadata/proto/v0/statistics.proto package v0 From dae055d5489ebaa1e6c9e24912a4beea7a658991 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 16 Oct 2020 18:19:31 +0800 Subject: [PATCH 17/24] Core: Update Core to support split StreamFormat and FileFormat messages Signed-off-by: Zhu Zhanyan --- .../java/feast/common/it/DataGenerator.java | 23 ++-- .../java/feast/core/model/DataSource.java | 95 ++++++++------- .../core/validators/DataSourceValidator.java | 68 ++++++----- .../java/feast/core/validators/Matchers.java | 10 ++ .../validators/DataSourceValidatorTest.java | 110 +++++++----------- 5 files changed, 157 insertions(+), 149 deletions(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index fe8ebf56e8c..66c1249004e 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -20,15 +20,16 @@ import com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import feast.proto.core.DataSourceProto.DataFormat; -import feast.proto.core.DataSourceProto.DataFormat.AvroFormat; -import feast.proto.core.DataSourceProto.DataFormat.ParquetFormat; -import feast.proto.core.DataSourceProto.DataFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; +import feast.proto.core.DataSourceProto.FileFormat; +import feast.proto.core.DataSourceProto.FileFormat.ParquetFormat; +import feast.proto.core.DataSourceProto.StreamFormat; +import feast.proto.core.DataSourceProto.StreamFormat.AvroFormat; +import feast.proto.core.DataSourceProto.StreamFormat.ProtoFormat; import feast.proto.core.EntityProto; import feast.proto.core.FeatureProto; import feast.proto.core.FeatureProto.FeatureSpecV2; @@ -344,24 +345,24 @@ public static DataSource createKinesisDataSourceSpec( KinesisOptions.newBuilder() .setRegion("ap-nowhere1") .setStreamName("stream") - .setRecordFormat(createProtoFormat("classPath")) + .setRecordFormat(createProtoFormat(classPath)) .build()) .setEventTimestampColumn(timestampColumn) .build(); } - public static DataFormat createParquetFormat() { - return DataFormat.newBuilder().setParquetFormat(ParquetFormat.getDefaultInstance()).build(); + public static FileFormat createParquetFormat() { + return FileFormat.newBuilder().setParquetFormat(ParquetFormat.getDefaultInstance()).build(); } - public static DataFormat createAvroFormat(String schemaJSON) { - return DataFormat.newBuilder() + public static StreamFormat createAvroFormat(String schemaJSON) { + return StreamFormat.newBuilder() .setAvroFormat(AvroFormat.newBuilder().setSchemaJson(schemaJSON).build()) .build(); } - public static DataFormat createProtoFormat(String classPath) { - return DataFormat.newBuilder() + public static StreamFormat createProtoFormat(String classPath) { + return StreamFormat.newBuilder() .setProtoFormat(ProtoFormat.newBuilder().setClassPath(classPath).build()) .build(); } diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java index 91513e21607..e60975da89f 100644 --- a/core/src/main/java/feast/core/model/DataSource.java +++ b/core/src/main/java/feast/core/model/DataSource.java @@ -17,15 +17,18 @@ package feast.core.model; import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; import feast.core.util.TypeConversion; import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataFormat; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import feast.proto.core.DataSourceProto.DataSource.SourceType; +import feast.proto.core.DataSourceProto.FileFormat; +import feast.proto.core.DataSourceProto.StreamFormat; import java.util.HashMap; import java.util.Map; import javax.persistence.Column; @@ -89,35 +92,30 @@ public static DataSource fromProto(DataSourceProto.DataSource spec) { DataSource source = new DataSource(spec.getType()); // Copy source type specific options Map dataSourceConfigMap = new HashMap<>(); - JsonFormat.Printer json = JsonFormat.printer(); - try { - switch (spec.getType()) { - case BATCH_FILE: - dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); - dataSourceConfigMap.put("file_format", json.print(spec.getFileOptions().getFileFormat())); - break; - case BATCH_BIGQUERY: - dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); - break; - case STREAM_KAFKA: - dataSourceConfigMap.put( - "bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); - dataSourceConfigMap.put( - "message_format", json.print(spec.getKafkaOptions().getMessageFormat())); - dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); - break; - case STREAM_KINESIS: - dataSourceConfigMap.put( - "record_format", json.print(spec.getKinesisOptions().getRecordFormat())); - dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); - dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); - } - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering DataFormat Proto to JSON", e); + switch (spec.getType()) { + case BATCH_FILE: + dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); + dataSourceConfigMap.put("file_format", printJSON(spec.getFileOptions().getFileFormat())); + break; + case BATCH_BIGQUERY: + dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); + break; + case STREAM_KAFKA: + dataSourceConfigMap.put("bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); + dataSourceConfigMap.put( + "message_format", printJSON(spec.getKafkaOptions().getMessageFormat())); + dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); + break; + case STREAM_KINESIS: + dataSourceConfigMap.put( + "record_format", printJSON(spec.getKinesisOptions().getRecordFormat())); + dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); + dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); + + break; + default: + throw new UnsupportedOperationException( + String.format("Unsupported Feature Store Type: %s", spec.getType())); } // Store DataSource mapping as serialised JSON @@ -146,7 +144,11 @@ public DataSourceProto.DataSource toProto() { case BATCH_FILE: FileOptions.Builder fileOptions = FileOptions.newBuilder(); fileOptions.setFileUrl(dataSourceConfigMap.get("file_url")); - fileOptions.setFileFormat(parseFormat(dataSourceConfigMap.get("file_format"))); + + FileFormat.Builder fileFormat = FileFormat.newBuilder(); + parseMessage(dataSourceConfigMap.get("file_format"), fileFormat); + fileOptions.setFileFormat(fileFormat.build()); + spec.setFileOptions(fileOptions.build()); break; case BATCH_BIGQUERY: @@ -158,14 +160,22 @@ public DataSourceProto.DataSource toProto() { KafkaOptions.Builder kafkaOptions = KafkaOptions.newBuilder(); kafkaOptions.setBootstrapServers(dataSourceConfigMap.get("bootstrap_servers")); kafkaOptions.setTopic(dataSourceConfigMap.get("topic")); - kafkaOptions.setMessageFormat(parseFormat(dataSourceConfigMap.get("message_format"))); + + StreamFormat.Builder messageFormat = StreamFormat.newBuilder(); + parseMessage(dataSourceConfigMap.get("message_format"), messageFormat); + kafkaOptions.setMessageFormat(messageFormat.build()); + spec.setKafkaOptions(kafkaOptions.build()); break; case STREAM_KINESIS: KinesisOptions.Builder kinesisOptions = KinesisOptions.newBuilder(); kinesisOptions.setRegion(dataSourceConfigMap.get("region")); kinesisOptions.setStreamName(dataSourceConfigMap.get("stream_name")); - kinesisOptions.setRecordFormat(parseFormat(dataSourceConfigMap.get("record_format"))); + + StreamFormat.Builder recordFormat = StreamFormat.newBuilder(); + parseMessage(dataSourceConfigMap.get("record_format"), recordFormat); + kinesisOptions.setRecordFormat(recordFormat.build()); + spec.setKinesisOptions(kinesisOptions.build()); break; default: @@ -204,14 +214,21 @@ public boolean equals(Object o) { return this.toProto().equals(other.toProto()); } - /** Parse the given data format in JSON representation to its protobuf representation. */ - private DataFormat parseFormat(String formatJSON) { + /** Print the given Message into its JSON string representation */ + private static String printJSON(MessageOrBuilder message) { + try { + return JsonFormat.printer().print(message); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException("Unexpected exception convering Proto to JSON", e); + } + } + + /** Parse the given Message in JSON representation into the given Message Builder */ + private static void parseMessage(String json, Message.Builder message) { try { - DataFormat.Builder format = DataFormat.newBuilder(); - JsonFormat.parser().merge(formatJSON, format); - return format.build(); + JsonFormat.parser().merge(json, message); } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering DataFormat JSON to Proto", e); + throw new RuntimeException("Unexpected exception convering JSON to Proto", e); } } } diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java index acf4a34e478..89ebf9f287f 100644 --- a/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ b/core/src/main/java/feast/core/validators/DataSourceValidator.java @@ -17,55 +17,65 @@ package feast.core.validators; import static feast.core.validators.Matchers.*; -import static feast.proto.core.DataSourceProto.DataFormat.FormatCase.*; import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; -import feast.proto.core.DataSourceProto.DataFormat.FormatCase; import feast.proto.core.DataSourceProto.DataSource; -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import java.util.Map; -import java.util.Set; +import feast.proto.core.DataSourceProto.FileFormat; +import feast.proto.core.DataSourceProto.StreamFormat; public class DataSourceValidator { - // Map data source type to its supported data formats - public static final Map> SUPPORTED_FORMATS = - Map.of( - BATCH_FILE, Set.of(PARQUET_FORMAT), - STREAM_KAFKA, Set.of(PROTO_FORMAT), - STREAM_KINESIS, Set.of(PROTO_FORMAT)); - /** Validate if the given DataSource protobuf spec is valid. */ public static void validate(DataSource spec) { switch (spec.getType()) { case BATCH_FILE: - // Verify that DataFormat is supported by file source - FormatCase fileFormat = spec.getFileOptions().getFileFormat().getFormatCase(); - validateFormat(spec.getType(), fileFormat); + FileFormat.FormatCase fileFormat = spec.getFileOptions().getFileFormat().getFormatCase(); + switch (fileFormat) { + case PARQUET_FORMAT: + break; + default: + throw new UnsupportedOperationException( + String.format("Unsupported File Format: %s", fileFormat)); + } break; + case BATCH_BIGQUERY: checkValidBigQueryTableRef(spec.getBigqueryOptions().getTableRef(), "FeatureTable"); break; + case STREAM_KAFKA: - // Verify that DataFormat is supported by kafka data source - FormatCase messageFormat = spec.getKafkaOptions().getMessageFormat().getFormatCase(); - validateFormat(spec.getType(), messageFormat); + StreamFormat.FormatCase messageFormat = + spec.getKafkaOptions().getMessageFormat().getFormatCase(); + switch (messageFormat) { + case PROTO_FORMAT: + checkValidClassPath( + spec.getKafkaOptions().getMessageFormat().getProtoFormat().getClassPath(), + "FeatureTable"); + break; + default: + throw new UnsupportedOperationException( + String.format( + "Unsupported Stream Format for Kafka Source Type: %s", messageFormat)); + } break; + case STREAM_KINESIS: - // Verify that DataFormat is supported by kinesis data source - FormatCase recordFormat = spec.getKinesisOptions().getRecordFormat().getFormatCase(); - validateFormat(spec.getType(), recordFormat); + // Verify tht DataFormat is supported by kinesis data source + StreamFormat.FormatCase recordFormat = + spec.getKinesisOptions().getRecordFormat().getFormatCase(); + switch (recordFormat) { + case PROTO_FORMAT: + checkValidClassPath( + spec.getKinesisOptions().getRecordFormat().getProtoFormat().getClassPath(), + "FeatureTable"); + break; + default: + throw new UnsupportedOperationException( + String.format("Unsupported Stream Format for Kafka Source Type: %s", recordFormat)); + } break; default: throw new UnsupportedOperationException( String.format("Unsupported Feature Store Type: %s", spec.getType())); } } - - /** Valdiates if the given data format is valid for the given data source type. */ - private static void validateFormat(SourceType sourceType, FormatCase format) { - if (!SUPPORTED_FORMATS.get(sourceType).contains(format)) { - throw new UnsupportedOperationException( - String.format("Unsupported Data Format for %s DataSource: %s", sourceType, format)); - } - } } diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java index dadc0a3b025..8ba89a8308a 100644 --- a/core/src/main/java/feast/core/validators/Matchers.java +++ b/core/src/main/java/feast/core/validators/Matchers.java @@ -24,6 +24,8 @@ public class Matchers { private static Pattern BIGQUERY_TABLE_REF_REGEX = Pattern.compile("[a-zA-Z0-9-]+[:]+[a-zA-Z0-9_]+[.]+[a-zA-Z0-9_]*"); + private static Pattern CLASS_PATH_REGEX = + Pattern.compile("[a-zA-Z_$][a-zA-Z0-9_$]*(\\.[a-zA-Z_$][a-zA-Z0-9_$]*)"); private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); @@ -92,6 +94,14 @@ public static void checkValidBigQueryTableRef(String input, String resource) } } + public static void checkValidClassPath(String input, String resource) { + if (!CLASS_PATH_REGEX.matcher(input).matches()) { + throw new IllegalArgumentException( + String.format( + ERROR_MESSAGE_TEMPLATE, resource, input, "argument must be a valid Java Classpath")); + } + } + public static boolean hasDuplicates(Collection strings) { return (new HashSet<>(strings)).size() < strings.size(); } diff --git a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java index 3bbd6e86d7e..842e4c657d6 100644 --- a/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java +++ b/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java @@ -16,18 +16,15 @@ */ package feast.core.validators; -import static feast.proto.core.DataSourceProto.DataFormat.FormatCase.*; import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; -import static org.junit.Assert.assertTrue; import feast.common.it.DataGenerator; import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataFormat.FormatCase; +import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; +import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; +import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import feast.proto.core.DataSourceProto.DataSource.SourceType; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.junit.Test; public class DataSourceValidatorTest { @@ -44,79 +41,52 @@ public void shouldPassValidSpecs() { getTestSpecsMap().values().forEach(DataSourceValidator::validate); } - public void shouldErrorOnUnsupportedDataFormats() { - - // Invert the supported formats to get a map of unsupported formats - Map> unsupportedFormatMap = - DataSourceValidator.SUPPORTED_FORMATS.entrySet().stream() - .map( - es -> { - Set supportedFormats = es.getValue(); - Set badFormats = - Stream.of(FormatCase.values()) - .filter(format -> !supportedFormats.contains(format)) - .collect(Collectors.toSet()); - return Map.entry(es.getKey(), badFormats); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + @Test(expected = IllegalArgumentException.class) + public void shouldErrorIfBadBigQueryTableRef() { + DataSourceProto.DataSource badSpec = + getTestSpecsMap() + .get(BATCH_BIGQUERY) + .toBuilder() + .setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef("bad:/ref").build()) + .build(); + DataSourceValidator.validate(badSpec); + } - // Check that validator should reject unsupported formats - unsupportedFormatMap.forEach( - (sourceType, badFormats) -> { - DataSourceProto.DataSource.Builder spec = getTestSpecsMap().get(sourceType).toBuilder(); - badFormats.forEach( - badFormat -> { - switch (sourceType) { - case BATCH_FILE: - spec.setFileOptions( - spec.getFileOptions() - .toBuilder() - .setFileFormat(getTestFormatsMap().get(badFormat))); - break; - case STREAM_KAFKA: - spec.setKafkaOptions( - spec.getKafkaOptions() - .toBuilder() - .setMessageFormat(getTestFormatsMap().get(badFormat))); - break; - case STREAM_KINESIS: - spec.setKinesisOptions( - spec.getKinesisOptions() - .toBuilder() - .setRecordFormat(getTestFormatsMap().get(badFormat))); - default: - throw new RuntimeException( - String.format("Unhandled data source type in test: %s", sourceType)); - } + @Test(expected = IllegalArgumentException.class) + public void shouldErrorIfBadClassPath() { + DataSourceProto.DataSource badSpec = + getTestSpecsMap() + .get(STREAM_KAFKA) + .toBuilder() + .setKafkaOptions( + KafkaOptions.newBuilder() + .setMessageFormat(DataGenerator.createProtoFormat(".bad^path")) + .build()) + .build(); + DataSourceValidator.validate(badSpec); - boolean hasError = false; - try { - DataSourceValidator.validate(spec.build()); - } catch (UnsupportedOperationException e) { - hasError = true; - } - assertTrue(hasError); - }); - }); + badSpec = + getTestSpecsMap() + .get(STREAM_KINESIS) + .toBuilder() + .setKinesisOptions( + KinesisOptions.newBuilder() + .setRecordFormat(DataGenerator.createProtoFormat(".bad^path")) + .build()) + .build(); + DataSourceValidator.validate(badSpec); } private Map getTestSpecsMap() { return Map.of( BATCH_FILE, DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), BATCH_BIGQUERY, - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col"), - STREAM_KINESIS, DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), - STREAM_KAFKA, + STREAM_KINESIS, DataGenerator.createKinesisDataSourceSpec( - "ap-nowhere1", "stream", "class.path", "ts_col")); - } - - private Map getTestFormatsMap() { - return Map.of( - PARQUET_FORMAT, DataGenerator.createParquetFormat(), - PROTO_FORMAT, DataGenerator.createProtoFormat("class.path"), - AVRO_FORMAT, DataGenerator.createAvroFormat("{}")); + "ap-nowhere1", "stream", "class.path", "ts_col"), + STREAM_KAFKA, + DataGenerator.createKafkaDataSourceSpec( + "localhost:9092", "topic", "class.path", "ts_col")); } } From e0d0e5dbb5f6df0efe2b548a4b09102fe7ce405c Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 18 Oct 2020 16:37:03 +0800 Subject: [PATCH 18/24] Proto: Split data formats from DataSource.proto into new DataFormat.proto Signed-off-by: Zhu Zhanyan --- .../java/feast/common/it/DataGenerator.java | 10 +- .../java/feast/core/model/DataSource.java | 4 +- .../core/validators/DataSourceValidator.java | 4 +- go.mod | 2 +- go.sum | 2 + protos/feast/core/DataFormat.proto | 56 ++ protos/feast/core/DataSource.proto | 33 +- sdk/go/protos/feast/core/DataFormat.pb.go | 494 ++++++++++++++ sdk/go/protos/feast/core/DataSource.pb.go | 645 ++++-------------- 9 files changed, 694 insertions(+), 556 deletions(-) create mode 100644 protos/feast/core/DataFormat.proto create mode 100644 sdk/go/protos/feast/core/DataFormat.pb.go diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 66c1249004e..9fe46bb3ba3 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -20,16 +20,16 @@ import com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; +import feast.proto.core.DataFormatProto.FileFormat; +import feast.proto.core.DataFormatProto.FileFormat.ParquetFormat; +import feast.proto.core.DataFormatProto.StreamFormat; +import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat; +import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.DataSourceProto.FileFormat; -import feast.proto.core.DataSourceProto.FileFormat.ParquetFormat; -import feast.proto.core.DataSourceProto.StreamFormat; -import feast.proto.core.DataSourceProto.StreamFormat.AvroFormat; -import feast.proto.core.DataSourceProto.StreamFormat.ProtoFormat; import feast.proto.core.EntityProto; import feast.proto.core.FeatureProto; import feast.proto.core.FeatureProto.FeatureSpecV2; diff --git a/core/src/main/java/feast/core/model/DataSource.java b/core/src/main/java/feast/core/model/DataSource.java index e60975da89f..67477dab45a 100644 --- a/core/src/main/java/feast/core/model/DataSource.java +++ b/core/src/main/java/feast/core/model/DataSource.java @@ -21,14 +21,14 @@ import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; import feast.core.util.TypeConversion; +import feast.proto.core.DataFormatProto.FileFormat; +import feast.proto.core.DataFormatProto.StreamFormat; import feast.proto.core.DataSourceProto; import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import feast.proto.core.DataSourceProto.DataSource.SourceType; -import feast.proto.core.DataSourceProto.FileFormat; -import feast.proto.core.DataSourceProto.StreamFormat; import java.util.HashMap; import java.util.Map; import javax.persistence.Column; diff --git a/core/src/main/java/feast/core/validators/DataSourceValidator.java b/core/src/main/java/feast/core/validators/DataSourceValidator.java index 89ebf9f287f..223906d0e60 100644 --- a/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ b/core/src/main/java/feast/core/validators/DataSourceValidator.java @@ -19,9 +19,9 @@ import static feast.core.validators.Matchers.*; import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; +import feast.proto.core.DataFormatProto.FileFormat; +import feast.proto.core.DataFormatProto.StreamFormat; import feast.proto.core.DataSourceProto.DataSource; -import feast.proto.core.DataSourceProto.FileFormat; -import feast.proto.core.DataSourceProto.StreamFormat; public class DataSourceValidator { /** Validate if the given DataSource protobuf spec is valid. */ diff --git a/go.mod b/go.mod index c7c3cbd9060..de70423b33e 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4 // indirect + golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.25.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 046c0ae5725..b9ec81936fc 100644 --- a/go.sum +++ b/go.sum @@ -498,6 +498,8 @@ golang.org/x/tools v0.0.0-20201013053347-2db1cd791039 h1:kLBxO4OPBgPwjg8Vvu+/0DC golang.org/x/tools v0.0.0-20201013053347-2db1cd791039/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4 h1:rQWkJiVIyJ3PgiSHL+RXc8xbrK8duU6jG5eeZ9G7nk8= golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 h1:ZB1XYzdDo7c/O48jzjMkvIjnC120Z9/CwgDWhePjQdQ= +golang.org/x/tools v0.0.0-20201017001424-6003fad69a88/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/protos/feast/core/DataFormat.proto b/protos/feast/core/DataFormat.proto new file mode 100644 index 00000000000..2926e08c630 --- /dev/null +++ b/protos/feast/core/DataFormat.proto @@ -0,0 +1,56 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "DataFormatProto"; +option java_package = "feast.proto.core"; + +// Defines the file format encoding the features/entity data in files +message FileFormat { + // Defines options for the Parquet data format + message ParquetFormat {} + + oneof format { + ParquetFormat parquet_format = 1; + } +} + +// Defines the data format encoding features/entity data in data streams +message StreamFormat { + // Defines options for the protobuf data format + message ProtoFormat { + // Classpath to the generated Java Protobuf class that can be used to decode + // Feature data from the obtained stream message + string class_path = 1; + } + + // Defines options for the avro data format + message AvroFormat { + // Optional if used in a File DataSource as schema is embedded in avro file. + // Specifies the schema of the Avro message as JSON string. + string schema_json = 1; + } + + // Specifies the data format and format specific options + oneof format { + AvroFormat avro_format = 1; + ProtoFormat proto_format = 2; + } +} diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 4f5fbf27476..c06a74b14c5 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -22,38 +22,7 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; option java_outer_classname = "DataSourceProto"; option java_package = "feast.proto.core"; -// Defines the file format encoding the features/entity data in files -message FileFormat { - // Defines options for the Parquet data format - message ParquetFormat {} - - oneof format { - ParquetFormat parquet_format = 1; - } -} - -// Defines the data format encoding features/entity data in data streams -message StreamFormat { - // Defines options for the protobuf data format - message ProtoFormat { - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained stream message - string class_path = 1; - } - - // Defines options for the avro data format - message AvroFormat { - // Optional if used in a File DataSource as schema is embedded in avro file. - // Specifies the schema of the Avro message as JSON string. - string schema_json = 1; - } - - // Specifies the data format and format specific options - oneof format { - AvroFormat avro_format = 1; - ProtoFormat proto_format = 2; - } -} +import "feast/core/DataFormat.proto"; // Defines a Data Source that can be used source Feature data message DataSource { diff --git a/sdk/go/protos/feast/core/DataFormat.pb.go b/sdk/go/protos/feast/core/DataFormat.pb.go new file mode 100644 index 00000000000..ac6d88f9c77 --- /dev/null +++ b/sdk/go/protos/feast/core/DataFormat.pb.go @@ -0,0 +1,494 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.10.0 +// source: feast/core/DataFormat.proto + +package core + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// Defines the file format encoding the features/entity data in files +type FileFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Format: + // *FileFormat_ParquetFormat_ + Format isFileFormat_Format `protobuf_oneof:"format"` +} + +func (x *FileFormat) Reset() { + *x = FileFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataFormat_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileFormat) ProtoMessage() {} + +func (x *FileFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataFormat_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileFormat.ProtoReflect.Descriptor instead. +func (*FileFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataFormat_proto_rawDescGZIP(), []int{0} +} + +func (m *FileFormat) GetFormat() isFileFormat_Format { + if m != nil { + return m.Format + } + return nil +} + +func (x *FileFormat) GetParquetFormat() *FileFormat_ParquetFormat { + if x, ok := x.GetFormat().(*FileFormat_ParquetFormat_); ok { + return x.ParquetFormat + } + return nil +} + +type isFileFormat_Format interface { + isFileFormat_Format() +} + +type FileFormat_ParquetFormat_ struct { + ParquetFormat *FileFormat_ParquetFormat `protobuf:"bytes,1,opt,name=parquet_format,json=parquetFormat,proto3,oneof"` +} + +func (*FileFormat_ParquetFormat_) isFileFormat_Format() {} + +// Defines the data format encoding features/entity data in data streams +type StreamFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Specifies the data format and format specific options + // + // Types that are assignable to Format: + // *StreamFormat_AvroFormat_ + // *StreamFormat_ProtoFormat_ + Format isStreamFormat_Format `protobuf_oneof:"format"` +} + +func (x *StreamFormat) Reset() { + *x = StreamFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataFormat_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat) ProtoMessage() {} + +func (x *StreamFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataFormat_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataFormat_proto_rawDescGZIP(), []int{1} +} + +func (m *StreamFormat) GetFormat() isStreamFormat_Format { + if m != nil { + return m.Format + } + return nil +} + +func (x *StreamFormat) GetAvroFormat() *StreamFormat_AvroFormat { + if x, ok := x.GetFormat().(*StreamFormat_AvroFormat_); ok { + return x.AvroFormat + } + return nil +} + +func (x *StreamFormat) GetProtoFormat() *StreamFormat_ProtoFormat { + if x, ok := x.GetFormat().(*StreamFormat_ProtoFormat_); ok { + return x.ProtoFormat + } + return nil +} + +type isStreamFormat_Format interface { + isStreamFormat_Format() +} + +type StreamFormat_AvroFormat_ struct { + AvroFormat *StreamFormat_AvroFormat `protobuf:"bytes,1,opt,name=avro_format,json=avroFormat,proto3,oneof"` +} + +type StreamFormat_ProtoFormat_ struct { + ProtoFormat *StreamFormat_ProtoFormat `protobuf:"bytes,2,opt,name=proto_format,json=protoFormat,proto3,oneof"` +} + +func (*StreamFormat_AvroFormat_) isStreamFormat_Format() {} + +func (*StreamFormat_ProtoFormat_) isStreamFormat_Format() {} + +// Defines options for the Parquet data format +type FileFormat_ParquetFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *FileFormat_ParquetFormat) Reset() { + *x = FileFormat_ParquetFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataFormat_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileFormat_ParquetFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileFormat_ParquetFormat) ProtoMessage() {} + +func (x *FileFormat_ParquetFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataFormat_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileFormat_ParquetFormat.ProtoReflect.Descriptor instead. +func (*FileFormat_ParquetFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataFormat_proto_rawDescGZIP(), []int{0, 0} +} + +// Defines options for the protobuf data format +type StreamFormat_ProtoFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Classpath to the generated Java Protobuf class that can be used to decode + // Feature data from the obtained stream message + ClassPath string `protobuf:"bytes,1,opt,name=class_path,json=classPath,proto3" json:"class_path,omitempty"` +} + +func (x *StreamFormat_ProtoFormat) Reset() { + *x = StreamFormat_ProtoFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataFormat_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat_ProtoFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat_ProtoFormat) ProtoMessage() {} + +func (x *StreamFormat_ProtoFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataFormat_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat_ProtoFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat_ProtoFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataFormat_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *StreamFormat_ProtoFormat) GetClassPath() string { + if x != nil { + return x.ClassPath + } + return "" +} + +// Defines options for the avro data format +type StreamFormat_AvroFormat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional if used in a File DataSource as schema is embedded in avro file. + // Specifies the schema of the Avro message as JSON string. + SchemaJson string `protobuf:"bytes,1,opt,name=schema_json,json=schemaJson,proto3" json:"schema_json,omitempty"` +} + +func (x *StreamFormat_AvroFormat) Reset() { + *x = StreamFormat_AvroFormat{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataFormat_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamFormat_AvroFormat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamFormat_AvroFormat) ProtoMessage() {} + +func (x *StreamFormat_AvroFormat) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataFormat_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamFormat_AvroFormat.ProtoReflect.Descriptor instead. +func (*StreamFormat_AvroFormat) Descriptor() ([]byte, []int) { + return file_feast_core_DataFormat_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *StreamFormat_AvroFormat) GetSchemaJson() string { + if x != nil { + return x.SchemaJson + } + return "" +} + +var File_feast_core_DataFormat_proto protoreflect.FileDescriptor + +var file_feast_core_DataFormat_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, + 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x76, 0x0a, 0x0a, 0x46, 0x69, 0x6c, + 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x4d, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x71, 0x75, + 0x65, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, + 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, + 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x0f, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x22, 0x88, 0x02, 0x0a, 0x0c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x12, 0x46, 0x0a, 0x0b, 0x61, 0x76, 0x72, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x2e, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0a, + 0x61, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x49, 0x0a, 0x0c, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x2c, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x46, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x50, + 0x61, 0x74, 0x68, 0x1a, 0x2d, 0x0a, 0x0a, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4a, 0x73, + 0x6f, 0x6e, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x58, 0x0a, 0x10, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, + 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_feast_core_DataFormat_proto_rawDescOnce sync.Once + file_feast_core_DataFormat_proto_rawDescData = file_feast_core_DataFormat_proto_rawDesc +) + +func file_feast_core_DataFormat_proto_rawDescGZIP() []byte { + file_feast_core_DataFormat_proto_rawDescOnce.Do(func() { + file_feast_core_DataFormat_proto_rawDescData = protoimpl.X.CompressGZIP(file_feast_core_DataFormat_proto_rawDescData) + }) + return file_feast_core_DataFormat_proto_rawDescData +} + +var file_feast_core_DataFormat_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_feast_core_DataFormat_proto_goTypes = []interface{}{ + (*FileFormat)(nil), // 0: feast.core.FileFormat + (*StreamFormat)(nil), // 1: feast.core.StreamFormat + (*FileFormat_ParquetFormat)(nil), // 2: feast.core.FileFormat.ParquetFormat + (*StreamFormat_ProtoFormat)(nil), // 3: feast.core.StreamFormat.ProtoFormat + (*StreamFormat_AvroFormat)(nil), // 4: feast.core.StreamFormat.AvroFormat +} +var file_feast_core_DataFormat_proto_depIdxs = []int32{ + 2, // 0: feast.core.FileFormat.parquet_format:type_name -> feast.core.FileFormat.ParquetFormat + 4, // 1: feast.core.StreamFormat.avro_format:type_name -> feast.core.StreamFormat.AvroFormat + 3, // 2: feast.core.StreamFormat.proto_format:type_name -> feast.core.StreamFormat.ProtoFormat + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_feast_core_DataFormat_proto_init() } +func file_feast_core_DataFormat_proto_init() { + if File_feast_core_DataFormat_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_feast_core_DataFormat_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataFormat_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataFormat_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileFormat_ParquetFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataFormat_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamFormat_ProtoFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataFormat_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamFormat_AvroFormat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_feast_core_DataFormat_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*FileFormat_ParquetFormat_)(nil), + } + file_feast_core_DataFormat_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*StreamFormat_AvroFormat_)(nil), + (*StreamFormat_ProtoFormat_)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_feast_core_DataFormat_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_feast_core_DataFormat_proto_goTypes, + DependencyIndexes: file_feast_core_DataFormat_proto_depIdxs, + MessageInfos: file_feast_core_DataFormat_proto_msgTypes, + }.Build() + File_feast_core_DataFormat_proto = out.File + file_feast_core_DataFormat_proto_rawDesc = nil + file_feast_core_DataFormat_proto_goTypes = nil + file_feast_core_DataFormat_proto_depIdxs = nil +} diff --git a/sdk/go/protos/feast/core/DataSource.pb.go b/sdk/go/protos/feast/core/DataSource.pb.go index b082e8bde6e..4dd43d51960 100644 --- a/sdk/go/protos/feast/core/DataSource.pb.go +++ b/sdk/go/protos/feast/core/DataSource.pb.go @@ -94,159 +94,9 @@ func (x DataSource_SourceType) Number() protoreflect.EnumNumber { // Deprecated: Use DataSource_SourceType.Descriptor instead. func (DataSource_SourceType) EnumDescriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 0} -} - -// Defines the file format encoding the features/entity data in files -type FileFormat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Format: - // *FileFormat_ParquetFormat_ - Format isFileFormat_Format `protobuf_oneof:"format"` -} - -func (x *FileFormat) Reset() { - *x = FileFormat{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FileFormat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FileFormat) ProtoMessage() {} - -func (x *FileFormat) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FileFormat.ProtoReflect.Descriptor instead. -func (*FileFormat) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0} -} - -func (m *FileFormat) GetFormat() isFileFormat_Format { - if m != nil { - return m.Format - } - return nil -} - -func (x *FileFormat) GetParquetFormat() *FileFormat_ParquetFormat { - if x, ok := x.GetFormat().(*FileFormat_ParquetFormat_); ok { - return x.ParquetFormat - } - return nil -} - -type isFileFormat_Format interface { - isFileFormat_Format() -} - -type FileFormat_ParquetFormat_ struct { - ParquetFormat *FileFormat_ParquetFormat `protobuf:"bytes,1,opt,name=parquet_format,json=parquetFormat,proto3,oneof"` -} - -func (*FileFormat_ParquetFormat_) isFileFormat_Format() {} - -// Defines the data format encoding features/entity data in data streams -type StreamFormat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Specifies the data format and format specific options - // - // Types that are assignable to Format: - // *StreamFormat_AvroFormat_ - // *StreamFormat_ProtoFormat_ - Format isStreamFormat_Format `protobuf_oneof:"format"` -} - -func (x *StreamFormat) Reset() { - *x = StreamFormat{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StreamFormat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamFormat) ProtoMessage() {} - -func (x *StreamFormat) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamFormat.ProtoReflect.Descriptor instead. -func (*StreamFormat) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1} -} - -func (m *StreamFormat) GetFormat() isStreamFormat_Format { - if m != nil { - return m.Format - } - return nil -} - -func (x *StreamFormat) GetAvroFormat() *StreamFormat_AvroFormat { - if x, ok := x.GetFormat().(*StreamFormat_AvroFormat_); ok { - return x.AvroFormat - } - return nil -} - -func (x *StreamFormat) GetProtoFormat() *StreamFormat_ProtoFormat { - if x, ok := x.GetFormat().(*StreamFormat_ProtoFormat_); ok { - return x.ProtoFormat - } - return nil -} - -type isStreamFormat_Format interface { - isStreamFormat_Format() -} - -type StreamFormat_AvroFormat_ struct { - AvroFormat *StreamFormat_AvroFormat `protobuf:"bytes,1,opt,name=avro_format,json=avroFormat,proto3,oneof"` -} - -type StreamFormat_ProtoFormat_ struct { - ProtoFormat *StreamFormat_ProtoFormat `protobuf:"bytes,2,opt,name=proto_format,json=protoFormat,proto3,oneof"` + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 0} } -func (*StreamFormat_AvroFormat_) isStreamFormat_Format() {} - -func (*StreamFormat_ProtoFormat_) isStreamFormat_Format() {} - // Defines a Data Source that can be used source Feature data type DataSource struct { state protoimpl.MessageState @@ -277,7 +127,7 @@ type DataSource struct { func (x *DataSource) Reset() { *x = DataSource{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[2] + mi := &file_feast_core_DataSource_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -290,7 +140,7 @@ func (x *DataSource) String() string { func (*DataSource) ProtoMessage() {} func (x *DataSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[2] + mi := &file_feast_core_DataSource_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -303,7 +153,7 @@ func (x *DataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource.ProtoReflect.Descriptor instead. func (*DataSource) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0} } func (x *DataSource) GetType() DataSource_SourceType { @@ -404,145 +254,6 @@ func (*DataSource_KafkaOptions_) isDataSource_Options() {} func (*DataSource_KinesisOptions_) isDataSource_Options() {} -// Defines options for the Parquet data format -type FileFormat_ParquetFormat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *FileFormat_ParquetFormat) Reset() { - *x = FileFormat_ParquetFormat{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FileFormat_ParquetFormat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FileFormat_ParquetFormat) ProtoMessage() {} - -func (x *FileFormat_ParquetFormat) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FileFormat_ParquetFormat.ProtoReflect.Descriptor instead. -func (*FileFormat_ParquetFormat) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 0} -} - -// Defines options for the protobuf data format -type StreamFormat_ProtoFormat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Classpath to the generated Java Protobuf class that can be used to decode - // Feature data from the obtained stream message - ClassPath string `protobuf:"bytes,1,opt,name=class_path,json=classPath,proto3" json:"class_path,omitempty"` -} - -func (x *StreamFormat_ProtoFormat) Reset() { - *x = StreamFormat_ProtoFormat{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StreamFormat_ProtoFormat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamFormat_ProtoFormat) ProtoMessage() {} - -func (x *StreamFormat_ProtoFormat) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamFormat_ProtoFormat.ProtoReflect.Descriptor instead. -func (*StreamFormat_ProtoFormat) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *StreamFormat_ProtoFormat) GetClassPath() string { - if x != nil { - return x.ClassPath - } - return "" -} - -// Defines options for the avro data format -type StreamFormat_AvroFormat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Optional if used in a File DataSource as schema is embedded in avro file. - // Specifies the schema of the Avro message as JSON string. - SchemaJson string `protobuf:"bytes,1,opt,name=schema_json,json=schemaJson,proto3" json:"schema_json,omitempty"` -} - -func (x *StreamFormat_AvroFormat) Reset() { - *x = StreamFormat_AvroFormat{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StreamFormat_AvroFormat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamFormat_AvroFormat) ProtoMessage() {} - -func (x *StreamFormat_AvroFormat) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamFormat_AvroFormat.ProtoReflect.Descriptor instead. -func (*StreamFormat_AvroFormat) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{1, 1} -} - -func (x *StreamFormat_AvroFormat) GetSchemaJson() string { - if x != nil { - return x.SchemaJson - } - return "" -} - // Defines options for DataSource that sources features from a file type DataSource_FileOptions struct { state protoimpl.MessageState @@ -560,7 +271,7 @@ type DataSource_FileOptions struct { func (x *DataSource_FileOptions) Reset() { *x = DataSource_FileOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[7] + mi := &file_feast_core_DataSource_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -573,7 +284,7 @@ func (x *DataSource_FileOptions) String() string { func (*DataSource_FileOptions) ProtoMessage() {} func (x *DataSource_FileOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[7] + mi := &file_feast_core_DataSource_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -586,7 +297,7 @@ func (x *DataSource_FileOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_FileOptions.ProtoReflect.Descriptor instead. func (*DataSource_FileOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 1} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 1} } func (x *DataSource_FileOptions) GetFileFormat() *FileFormat { @@ -616,7 +327,7 @@ type DataSource_BigQueryOptions struct { func (x *DataSource_BigQueryOptions) Reset() { *x = DataSource_BigQueryOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[8] + mi := &file_feast_core_DataSource_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -629,7 +340,7 @@ func (x *DataSource_BigQueryOptions) String() string { func (*DataSource_BigQueryOptions) ProtoMessage() {} func (x *DataSource_BigQueryOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[8] + mi := &file_feast_core_DataSource_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -642,7 +353,7 @@ func (x *DataSource_BigQueryOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_BigQueryOptions.ProtoReflect.Descriptor instead. func (*DataSource_BigQueryOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 2} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 2} } func (x *DataSource_BigQueryOptions) GetTableRef() string { @@ -671,7 +382,7 @@ type DataSource_KafkaOptions struct { func (x *DataSource_KafkaOptions) Reset() { *x = DataSource_KafkaOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[9] + mi := &file_feast_core_DataSource_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -684,7 +395,7 @@ func (x *DataSource_KafkaOptions) String() string { func (*DataSource_KafkaOptions) ProtoMessage() {} func (x *DataSource_KafkaOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[9] + mi := &file_feast_core_DataSource_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -697,7 +408,7 @@ func (x *DataSource_KafkaOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_KafkaOptions.ProtoReflect.Descriptor instead. func (*DataSource_KafkaOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 3} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 3} } func (x *DataSource_KafkaOptions) GetBootstrapServers() string { @@ -741,7 +452,7 @@ type DataSource_KinesisOptions struct { func (x *DataSource_KinesisOptions) Reset() { *x = DataSource_KinesisOptions{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_DataSource_proto_msgTypes[10] + mi := &file_feast_core_DataSource_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -754,7 +465,7 @@ func (x *DataSource_KinesisOptions) String() string { func (*DataSource_KinesisOptions) ProtoMessage() {} func (x *DataSource_KinesisOptions) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_DataSource_proto_msgTypes[10] + mi := &file_feast_core_DataSource_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -767,7 +478,7 @@ func (x *DataSource_KinesisOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DataSource_KinesisOptions.ProtoReflect.Descriptor instead. func (*DataSource_KinesisOptions) Descriptor() ([]byte, []int) { - return file_feast_core_DataSource_proto_rawDescGZIP(), []int{2, 4} + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 4} } func (x *DataSource_KinesisOptions) GetRegion() string { @@ -796,115 +507,93 @@ var File_feast_core_DataSource_proto protoreflect.FileDescriptor var file_feast_core_DataSource_proto_rawDesc = []byte{ 0x0a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x76, 0x0a, 0x0a, 0x46, 0x69, 0x6c, - 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x4d, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x71, 0x75, - 0x65, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x46, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, - 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x0f, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, - 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x22, 0x88, 0x02, 0x0a, 0x0c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, - 0x61, 0x74, 0x12, 0x46, 0x0a, 0x0b, 0x61, 0x76, 0x72, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd6, 0x09, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x0d, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x16, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, + 0x47, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0b, 0x66, 0x69, 0x6c, + 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, 0x10, 0x62, 0x69, 0x67, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0f, 0x62, 0x69, + 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, + 0x0d, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x61, 0x66, + 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6b, 0x61, 0x66, + 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x0f, 0x6b, 0x69, 0x6e, + 0x65, 0x73, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x65, 0x73, + 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0e, 0x6b, 0x69, 0x6e, + 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x61, 0x0a, 0x0b, + 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x1a, + 0x2e, 0x0a, 0x0f, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x66, 0x1a, + 0x92, 0x01, 0x0a, 0x0c, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, + 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, + 0x70, 0x69, 0x63, 0x12, 0x3f, 0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x46, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x88, 0x01, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x3d, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x2e, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0a, - 0x61, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x49, 0x0a, 0x0c, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x46, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x2c, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x50, - 0x61, 0x74, 0x68, 0x1a, 0x2d, 0x0a, 0x0a, 0x41, 0x76, 0x72, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4a, 0x73, - 0x6f, 0x6e, 0x42, 0x08, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd6, 0x09, 0x0a, - 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x0d, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x70, 0x70, - 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x12, 0x34, 0x0a, 0x16, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x14, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, 0x74, 0x65, 0x5f, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x47, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, - 0x00, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, - 0x0a, 0x10, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x48, 0x00, 0x52, 0x0f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, - 0x00, 0x52, 0x0c, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x50, 0x0a, 0x0f, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x4b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, - 0x00, 0x52, 0x0e, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x61, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, - 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, - 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, - 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x1a, 0x2e, 0x0a, 0x0f, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, - 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x52, 0x65, 0x66, 0x1a, 0x92, 0x01, 0x0a, 0x0c, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, - 0x72, 0x61, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x3f, 0x0a, 0x0e, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0d, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, 0x88, 0x01, 0x0a, 0x0e, 0x4b, - 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, - 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x46, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0x63, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, - 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, - 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, - 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, - 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, - 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, 0x49, 0x53, 0x10, 0x04, 0x42, 0x09, 0x0a, 0x07, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, - 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, + 0x63, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, + 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, + 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, + 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, + 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, + 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, + 0x49, 0x53, 0x10, 0x04, 0x42, 0x09, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, + 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -920,39 +609,33 @@ func file_feast_core_DataSource_proto_rawDescGZIP() []byte { } var file_feast_core_DataSource_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_feast_core_DataSource_proto_goTypes = []interface{}{ (DataSource_SourceType)(0), // 0: feast.core.DataSource.SourceType - (*FileFormat)(nil), // 1: feast.core.FileFormat - (*StreamFormat)(nil), // 2: feast.core.StreamFormat - (*DataSource)(nil), // 3: feast.core.DataSource - (*FileFormat_ParquetFormat)(nil), // 4: feast.core.FileFormat.ParquetFormat - (*StreamFormat_ProtoFormat)(nil), // 5: feast.core.StreamFormat.ProtoFormat - (*StreamFormat_AvroFormat)(nil), // 6: feast.core.StreamFormat.AvroFormat - nil, // 7: feast.core.DataSource.FieldMappingEntry - (*DataSource_FileOptions)(nil), // 8: feast.core.DataSource.FileOptions - (*DataSource_BigQueryOptions)(nil), // 9: feast.core.DataSource.BigQueryOptions - (*DataSource_KafkaOptions)(nil), // 10: feast.core.DataSource.KafkaOptions - (*DataSource_KinesisOptions)(nil), // 11: feast.core.DataSource.KinesisOptions + (*DataSource)(nil), // 1: feast.core.DataSource + nil, // 2: feast.core.DataSource.FieldMappingEntry + (*DataSource_FileOptions)(nil), // 3: feast.core.DataSource.FileOptions + (*DataSource_BigQueryOptions)(nil), // 4: feast.core.DataSource.BigQueryOptions + (*DataSource_KafkaOptions)(nil), // 5: feast.core.DataSource.KafkaOptions + (*DataSource_KinesisOptions)(nil), // 6: feast.core.DataSource.KinesisOptions + (*FileFormat)(nil), // 7: feast.core.FileFormat + (*StreamFormat)(nil), // 8: feast.core.StreamFormat } var file_feast_core_DataSource_proto_depIdxs = []int32{ - 4, // 0: feast.core.FileFormat.parquet_format:type_name -> feast.core.FileFormat.ParquetFormat - 6, // 1: feast.core.StreamFormat.avro_format:type_name -> feast.core.StreamFormat.AvroFormat - 5, // 2: feast.core.StreamFormat.proto_format:type_name -> feast.core.StreamFormat.ProtoFormat - 0, // 3: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType - 7, // 4: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry - 8, // 5: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions - 9, // 6: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions - 10, // 7: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions - 11, // 8: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions - 1, // 9: feast.core.DataSource.FileOptions.file_format:type_name -> feast.core.FileFormat - 2, // 10: feast.core.DataSource.KafkaOptions.message_format:type_name -> feast.core.StreamFormat - 2, // 11: feast.core.DataSource.KinesisOptions.record_format:type_name -> feast.core.StreamFormat - 12, // [12:12] is the sub-list for method output_type - 12, // [12:12] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 0, // 0: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType + 2, // 1: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry + 3, // 2: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions + 4, // 3: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions + 5, // 4: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions + 6, // 5: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions + 7, // 6: feast.core.DataSource.FileOptions.file_format:type_name -> feast.core.FileFormat + 8, // 7: feast.core.DataSource.KafkaOptions.message_format:type_name -> feast.core.StreamFormat + 8, // 8: feast.core.DataSource.KinesisOptions.record_format:type_name -> feast.core.StreamFormat + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_feast_core_DataSource_proto_init() } @@ -960,32 +643,9 @@ func file_feast_core_DataSource_proto_init() { if File_feast_core_DataSource_proto != nil { return } + file_feast_core_DataFormat_proto_init() if !protoimpl.UnsafeEnabled { file_feast_core_DataSource_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileFormat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_DataSource_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StreamFormat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_DataSource_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource); i { case 0: return &v.state @@ -997,43 +657,7 @@ func file_feast_core_DataSource_proto_init() { return nil } } - file_feast_core_DataSource_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileFormat_ParquetFormat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_DataSource_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StreamFormat_ProtoFormat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_DataSource_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StreamFormat_AvroFormat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_DataSource_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_DataSource_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource_FileOptions); i { case 0: return &v.state @@ -1045,7 +669,7 @@ func file_feast_core_DataSource_proto_init() { return nil } } - file_feast_core_DataSource_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_DataSource_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource_BigQueryOptions); i { case 0: return &v.state @@ -1057,7 +681,7 @@ func file_feast_core_DataSource_proto_init() { return nil } } - file_feast_core_DataSource_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_DataSource_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource_KafkaOptions); i { case 0: return &v.state @@ -1069,7 +693,7 @@ func file_feast_core_DataSource_proto_init() { return nil } } - file_feast_core_DataSource_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_DataSource_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataSource_KinesisOptions); i { case 0: return &v.state @@ -1083,13 +707,6 @@ func file_feast_core_DataSource_proto_init() { } } file_feast_core_DataSource_proto_msgTypes[0].OneofWrappers = []interface{}{ - (*FileFormat_ParquetFormat_)(nil), - } - file_feast_core_DataSource_proto_msgTypes[1].OneofWrappers = []interface{}{ - (*StreamFormat_AvroFormat_)(nil), - (*StreamFormat_ProtoFormat_)(nil), - } - file_feast_core_DataSource_proto_msgTypes[2].OneofWrappers = []interface{}{ (*DataSource_FileOptions_)(nil), (*DataSource_BigqueryOptions)(nil), (*DataSource_KafkaOptions_)(nil), @@ -1101,7 +718,7 @@ func file_feast_core_DataSource_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_DataSource_proto_rawDesc, NumEnums: 1, - NumMessages: 11, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, From eff0115df180dc186e4677dfe912bff3cc40caa4 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 18 Oct 2020 17:25:45 +0800 Subject: [PATCH 19/24] Python SDK: Update SDK to support split StreamFormat and FileFormat messages Signed-off-by: Zhu Zhanyan --- sdk/python/feast/data_source.py | 111 +++++-------------------- sdk/python/tests/test_feature_table.py | 3 +- tests/e2e/test-register.py | 2 + 3 files changed, 23 insertions(+), 93 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index a3ed485f904..1eea7a53e6d 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -17,10 +17,15 @@ from abc import ABC, abstractmethod from typing import Dict, Optional -from feast.core.DataSource_pb2 import DataFormat as DataFormatProto +from feast.data_format import ( + FileFormat, + StreamFormat, + ProtoFormat, + AvroFormat, + ParquetFormat, +) from feast.core.DataSource_pb2 import DataSource as DataSourceProto - class SourceType(enum.Enum): """ DataSource value type. Used to define source types in DataSource. @@ -32,91 +37,13 @@ class SourceType(enum.Enum): STREAM_KAFKA = 3 STREAM_KINESIS = 4 - -class DataFormat(ABC): - """ - Defines an abstract data format used to encode feature data. - """ - - @abstractmethod - def to_proto(self): - """ - Convert this DataFormat into its protobuf representation. - """ - pass - - def __eq__(self, other): - return self.to_proto() == other.to_proto() - - @classmethod - def from_proto(cls, proto): - """ - Construct this DataFormat from its protobuf representation. - """ - fmt = proto.WhichOneof("format") - if fmt == "avro_format": - return AvroFormat(schema_json=proto.avro_format.schema_json) - if fmt == "parquet_format": - return ParquetFormat() - if fmt == "proto_format": - return ProtoFormat(class_path=proto.proto_format.class_path) - - -class AvroFormat(DataFormat): - """ - Defines the Avro data format - """ - - def __init__(self, schema_json: str): - """ - Construct a new Avro data format. - - Args: - schema_json: Avro schema definition in JSON - """ - self.schema_json = schema_json - - def to_proto(self): - proto = DataFormatProto.AvroFormat(schema_json=self.schema_json) - return DataFormatProto(avro_format=proto) - - -class ParquetFormat(DataFormat): - """ - Defines the Parquet data format - """ - - def to_proto(self): - return DataFormatProto(parquet_format=DataFormatProto.ParquetFormat()) - - -class ProtoFormat(DataFormat): - """ - Defines the Protobuf data format - """ - - def __init__(self, class_path: str): - """ - Construct a new Protobuf data format. - - Args: - class_path: Class path to the Java Protobuf class that can be used to decode protobuf messages.; - """ - self.class_path = class_path - - def to_proto(self): - return DataFormatProto( - proto_format=DataFormatProto.ProtoFormat(class_path=self.class_path) - ) - - class FileOptions: """ DataSource File options used to source features from a file """ def __init__( - self, file_format: DataFormat, file_url: str, + self, file_format: FileFormat, file_url: str, ): self._file_format = file_format self._file_url = file_url @@ -161,7 +88,7 @@ def from_proto(cls, file_options_proto: DataSourceProto.FileOptions): Returns a FileOptions object based on the file_options protobuf """ file_options = cls( - file_format=DataFormat.from_proto(file_options_proto.file_format), + file_format=FileFormat.from_proto(file_options_proto.file_format), file_url=file_options_proto.file_url, ) return file_options @@ -242,7 +169,7 @@ class KafkaOptions: """ def __init__( - self, bootstrap_servers: str, message_format: DataFormat, topic: str, + self, bootstrap_servers: str, message_format: StreamFormat, topic: str, ): self._bootstrap_servers = bootstrap_servers self._message_format = message_format @@ -304,7 +231,7 @@ def from_proto(cls, kafka_options_proto: DataSourceProto.KafkaOptions): kafka_options = cls( bootstrap_servers=kafka_options_proto.bootstrap_servers, - message_format=DataFormat.from_proto(kafka_options_proto.message_format), + message_format=StreamFormat.from_proto(kafka_options_proto.message_format), topic=kafka_options_proto.topic, ) @@ -333,7 +260,7 @@ class KinesisOptions: """ def __init__( - self, record_format: DataFormat, region: str, stream_name: str, + self, record_format: StreamFormat, region: str, stream_name: str, ): self._record_format = record_format self._region = region @@ -394,7 +321,7 @@ def from_proto(cls, kinesis_options_proto: DataSourceProto.KinesisOptions): """ kinesis_options = cls( - record_format=DataFormat.from_proto(kinesis_options_proto.record_format), + record_format=StreamFormat.from_proto(kinesis_options_proto.record_format), region=kinesis_options_proto.region, stream_name=kinesis_options_proto.stream_name, ) @@ -514,7 +441,7 @@ def from_proto(data_source): if data_source.file_options.file_format and data_source.file_options.file_url: data_source_obj = FileSource( field_mapping=data_source.field_mapping, - file_format=DataFormat.from_proto(data_source.file_options.file_format), + file_format=FileFormat.from_proto(data_source.file_options.file_format), file_url=data_source.file_options.file_url, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, @@ -536,7 +463,7 @@ def from_proto(data_source): data_source_obj = KafkaSource( field_mapping=data_source.field_mapping, bootstrap_servers=data_source.kafka_options.bootstrap_servers, - message_format=DataFormat.from_proto( + message_format=StreamFormat.from_proto( data_source.kafka_options.message_format ), topic=data_source.kafka_options.topic, @@ -551,7 +478,7 @@ def from_proto(data_source): ): data_source_obj = KinesisSource( field_mapping=data_source.field_mapping, - record_format=DataFormat.from_proto( + record_format=StreamFormat.from_proto( data_source.kinesis_options.record_format ), region=data_source.kinesis_options.region, @@ -577,7 +504,7 @@ def __init__( self, event_timestamp_column: str, created_timestamp_column: str, - file_format: DataFormat, + file_format: FileFormat, file_url: str, field_mapping: Optional[Dict[str, str]] = dict(), date_partition_column: Optional[str] = "", @@ -691,7 +618,7 @@ def __init__( event_timestamp_column: str, created_timestamp_column: str, bootstrap_servers: str, - message_format: DataFormat, + message_format: StreamFormat, topic: str, field_mapping: Optional[Dict[str, str]] = dict(), date_partition_column: Optional[str] = "", @@ -757,7 +684,7 @@ def __init__( self, event_timestamp_column: str, created_timestamp_column: str, - record_format: DataFormat, + record_format: StreamFormat, region: str, stream_name: str, field_mapping: Optional[Dict[str, str]] = dict(), diff --git a/sdk/python/tests/test_feature_table.py b/sdk/python/tests/test_feature_table.py index 0330ae6158e..3ab836463fd 100644 --- a/sdk/python/tests/test_feature_table.py +++ b/sdk/python/tests/test_feature_table.py @@ -21,7 +21,8 @@ from feast.client import Client from feast.core import CoreService_pb2_grpc as Core -from feast.data_source import FileSource, KafkaSource, ParquetFormat, ProtoFormat +from feast.data_source import FileSource, KafkaSource +from feast.data_format import ParquetFormat, ProtoFormat from feast.feature import Feature from feast.feature_table import FeatureTable from feast.value_type import ValueType diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index 4eb67ad949f..bbd310a4a2b 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -14,6 +14,8 @@ BigQuerySource, FileSource, KafkaSource, +) +from feast.data_format import ( ParquetFormat, ProtoFormat, ) From 02a4924310e0054cca242801f06bf3e1bdefcc34 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 18 Oct 2020 17:40:59 +0800 Subject: [PATCH 20/24] Python SDK: Fix python lint Signed-off-by: Zhu Zhanyan --- sdk/python/feast/client.py | 3 ++- sdk/python/feast/data_source.py | 11 +++-------- sdk/python/tests/test_client.py | 3 ++- sdk/python/tests/test_feature_table.py | 2 +- tests/e2e/test-register.py | 11 ++--------- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 0b73c6ad459..08d6b5a9be8 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -58,7 +58,8 @@ ListProjectsResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.data_source import BigQuerySource, FileSource, ParquetFormat +from feast.data_format import ParquetFormat +from feast.data_source import BigQuerySource, FileSource from feast.entity import Entity from feast.feature import _build_feature_references from feast.feature_table import FeatureTable diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 1eea7a53e6d..58f7ec6bdb7 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,17 +14,11 @@ import enum -from abc import ABC, abstractmethod from typing import Dict, Optional -from feast.data_format import ( - FileFormat, - StreamFormat, - ProtoFormat, - AvroFormat, - ParquetFormat, -) from feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.data_format import FileFormat, StreamFormat + class SourceType(enum.Enum): """ @@ -37,6 +31,7 @@ class SourceType(enum.Enum): STREAM_KAFKA = 3 STREAM_KINESIS = 4 + class FileOptions: """ DataSource File options used to source features from a file diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 147a8bbe7cd..54948c273b4 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -41,7 +41,8 @@ from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto from feast.core.FeatureTable_pb2 import FeatureTableMeta as FeatureTableMetaProto from feast.core.FeatureTable_pb2 import FeatureTableSpec as FeatureTableSpecProto -from feast.data_source import FileSource, KafkaSource, ParquetFormat, ProtoFormat +from feast.data_format import ParquetFormat, ProtoFormat +from feast.data_source import FileSource, KafkaSource from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable diff --git a/sdk/python/tests/test_feature_table.py b/sdk/python/tests/test_feature_table.py index 3ab836463fd..858704bd7c5 100644 --- a/sdk/python/tests/test_feature_table.py +++ b/sdk/python/tests/test_feature_table.py @@ -21,8 +21,8 @@ from feast.client import Client from feast.core import CoreService_pb2_grpc as Core -from feast.data_source import FileSource, KafkaSource from feast.data_format import ParquetFormat, ProtoFormat +from feast.data_source import FileSource, KafkaSource from feast.feature import Feature from feast.feature_table import FeatureTable from feast.value_type import ValueType diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py index bbd310a4a2b..4bd8966817b 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test-register.py @@ -10,15 +10,8 @@ from pandas.testing import assert_frame_equal from feast.client import Client -from feast.data_source import ( - BigQuerySource, - FileSource, - KafkaSource, -) -from feast.data_format import ( - ParquetFormat, - ProtoFormat, -) +from feast.data_format import ParquetFormat, ProtoFormat +from feast.data_source import BigQuerySource, FileSource, KafkaSource from feast.entity import Entity from feast.feature import Feature from feast.feature_table import FeatureTable From 252d64b5a799ab9439795cf8fe6271aea110f791 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 19 Oct 2020 09:18:26 +0800 Subject: [PATCH 21/24] Python SDK: Fix missing data_format module due to file being untracked by git Signed-off-by: Zhu Zhanyan --- sdk/python/feast/data_format.py | 121 ++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 sdk/python/feast/data_format.py diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py new file mode 100644 index 00000000000..3edaf3b3f32 --- /dev/null +++ b/sdk/python/feast/data_format.py @@ -0,0 +1,121 @@ +# Copyright 2020 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import enum +from abc import ABC, abstractmethod +from typing import Dict, Optional + +from feast.core.DataFormat_pb2 import ( + StreamFormat as StreamFormatProto, + FileFormat as FileFormatProto, +) + +class FileFormat(ABC): + """ + Defines an abtract file forma used to encode feature data in filesf + """ + @abstractmethod + def to_proto(self): + """ + Convert this FileFormat into its protobuf representation. + """ + pass + + def __eq__(self, other): + return self.to_proto() == other.to_proto() + + @classmethod + def from_proto(cls, proto): + """ + Construct this FileFormat from its protobuf representation. + Raises NotImplementedError if FileFormat specified in given proto is not supported. + """ + fmt = proto.WhichOneof("format") + if fmt == "parquet_format": + return ParquetFormat() + raise NotImplementedError(f"FileFormat is unsupported: {fmt}") + +class ParquetFormat(FileFormat): + """ + Defines the Parquet data format + """ + + def to_proto(self): + return FileFormatProto(parquet_format=FileFormatProto.ParquetFormat()) + +class StreamFormat(ABC): + """ + Defines an abtracts streaming data format used to encode feature data in streams + """ + + @abstractmethod + def to_proto(self): + """ + Convert this StreamFormat into its protobuf representation. + """ + pass + + def __eq__(self, other): + return self.to_proto() == other.to_proto() + + @classmethod + def from_proto(cls, proto): + """ + Construct this StreamFormat from its protobuf representation. + """ + fmt = proto.WhichOneof("format") + if fmt == "avro_format": + return AvroFormat(schema_json=proto.avro_format.schema_json) + if fmt == "proto_format": + return ProtoFormat(class_path=proto.proto_format.class_path) + raise NotImplementedError(f"StreamFormat is unsupported: {fmt}") + +class AvroFormat(StreamFormat): + """ + Defines the Avro streaming data format that encodes data in Avro format + """ + + def __init__(self, schema_json: str): + """ + Construct a new Avro data format. + + Args: + schema_json: Avro schema definition in JSON + """ + self.schema_json = schema_json + + def to_proto(self): + proto = StreamFormatProto.AvroFormat(schema_json=self.schema_json) + return StreamFormatProto(avro_format=proto) + + +class ProtoFormat(StreamFormat): + """ + Defines the Protobuf data format + """ + + def __init__(self, class_path: str): + """ + Construct a new Protobuf data format. + + Args: + class_path: Class path to the Java Protobuf class that can be used to decode protobuf messages.; + """ + self.class_path = class_path + + def to_proto(self): + return StreamFormatProto( + proto_format=StreamFormatProto.ProtoFormat(class_path=self.class_path) + ) From 2dabb3a39713bdc7086d7bffdf713c2e3f97a75a Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 19 Oct 2020 09:47:49 +0800 Subject: [PATCH 22/24] Python SDK: Fix Lint Signed-off-by: Zhu Zhanyan --- sdk/python/feast/data_format.py | 13 +++++++------ sdk/python/feast/pyspark/launchers/aws/emr.py | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py index 3edaf3b3f32..a8980f6a5c8 100644 --- a/sdk/python/feast/data_format.py +++ b/sdk/python/feast/data_format.py @@ -13,19 +13,17 @@ # limitations under the License. -import enum from abc import ABC, abstractmethod -from typing import Dict, Optional -from feast.core.DataFormat_pb2 import ( - StreamFormat as StreamFormatProto, - FileFormat as FileFormatProto, -) +from feast.core.DataFormat_pb2 import FileFormat as FileFormatProto +from feast.core.DataFormat_pb2 import StreamFormat as StreamFormatProto + class FileFormat(ABC): """ Defines an abtract file forma used to encode feature data in filesf """ + @abstractmethod def to_proto(self): """ @@ -47,6 +45,7 @@ def from_proto(cls, proto): return ParquetFormat() raise NotImplementedError(f"FileFormat is unsupported: {fmt}") + class ParquetFormat(FileFormat): """ Defines the Parquet data format @@ -55,6 +54,7 @@ class ParquetFormat(FileFormat): def to_proto(self): return FileFormatProto(parquet_format=FileFormatProto.ParquetFormat()) + class StreamFormat(ABC): """ Defines an abtracts streaming data format used to encode feature data in streams @@ -82,6 +82,7 @@ def from_proto(cls, proto): return ProtoFormat(class_path=proto.proto_format.class_path) raise NotImplementedError(f"StreamFormat is unsupported: {fmt}") + class AvroFormat(StreamFormat): """ Defines the Avro streaming data format that encodes data in Avro format diff --git a/sdk/python/feast/pyspark/launchers/aws/emr.py b/sdk/python/feast/pyspark/launchers/aws/emr.py index 0e8a4c23a87..04b32ac9231 100644 --- a/sdk/python/feast/pyspark/launchers/aws/emr.py +++ b/sdk/python/feast/pyspark/launchers/aws/emr.py @@ -6,6 +6,7 @@ import boto3 import pandas +from feast.data_format import ParquetFormat from feast.data_source import FileSource from feast.pyspark.abc import ( IngestionJob, @@ -242,6 +243,6 @@ def stage_dataframe( return FileSource( event_timestamp_column=event_timestamp, created_timestamp_column=created_timestamp_column, - file_format="parquet", + file_format=ParquetFormat(), file_url=file_url, ) From 1bb704e8440870718129c06bdbc3ab90ce4948ff Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 19 Oct 2020 10:08:36 +0800 Subject: [PATCH 23/24] Python SDK: Fix data_format specification format Signed-off-by: Zhu Zhanyan --- sdk/python/tests/test_historical_feature_retrieval.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/python/tests/test_historical_feature_retrieval.py b/sdk/python/tests/test_historical_feature_retrieval.py index d9dbd9f4fcb..286661148a1 100644 --- a/sdk/python/tests/test_historical_feature_retrieval.py +++ b/sdk/python/tests/test_historical_feature_retrieval.py @@ -22,6 +22,7 @@ from feast import Client, Entity, Feature, FeatureTable, FileSource, ValueType from feast.core import CoreService_pb2_grpc as Core +from feast.data_format import ParquetFormat from tests.feast_core_server import CoreServicer @@ -153,7 +154,7 @@ def transactions_feature_table(spark, client): spark, "transactions", schema, df_data ) file_source = FileSource( - "event_timestamp", "created_timestamp", "parquet", file_uri + "event_timestamp", "created_timestamp", ParquetFormat(), file_uri ) features = [ Feature("total_transactions", ValueType.DOUBLE), @@ -199,7 +200,7 @@ def bookings_feature_table(spark, client): temp_dir, file_uri = create_temp_parquet_file(spark, "bookings", schema, df_data) file_source = FileSource( - "event_timestamp", "created_timestamp", "parquet", file_uri + "event_timestamp", "created_timestamp", ParquetFormat(), file_uri ) features = [Feature("total_completed_bookings", ValueType.INT32)] max_age = Duration() @@ -244,7 +245,7 @@ def bookings_feature_table_with_mapping(spark, client): temp_dir, file_uri = create_temp_parquet_file(spark, "bookings", schema, df_data) file_source = FileSource( - "datetime", "created_datetime", "parquet", file_uri, {"id": "driver_id"} + "datetime", "created_datetime", ParquetFormat(), file_uri, {"id": "driver_id"} ) features = [Feature("total_completed_bookings", ValueType.INT32)] max_age = Duration() @@ -283,7 +284,7 @@ def test_historical_feature_retrieval_from_local_spark_session( spark, "customer_driver_pair", schema, df_data ) customer_driver_pairs_source = FileSource( - "event_timestamp", "created_timestamp", "parquet", file_uri + "event_timestamp", "created_timestamp", ParquetFormat(), file_uri ) joined_df = client.get_historical_features_df( ["transactions:total_transactions", "bookings:total_completed_bookings"], @@ -330,7 +331,7 @@ def test_historical_feature_retrieval_with_field_mappings_from_local_spark_sessi ] temp_dir, file_uri = create_temp_parquet_file(spark, "drivers", schema, df_data) entity_source = FileSource( - "event_timestamp", "created_timestamp", "parquet", file_uri + "event_timestamp", "created_timestamp", ParquetFormat(), file_uri ) joined_df = client.get_historical_features_df( ["bookings:total_completed_bookings"], entity_source, From dc872a4836563239c68defca89ce6524998c292f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 19 Oct 2020 11:29:37 +0800 Subject: [PATCH 24/24] Python SDK: Fix issue where _source_to_argument pass FileFormat class instead of expected str Signed-off-by: Zhu Zhanyan --- sdk/python/feast/data_format.py | 13 +++++++++++-- sdk/python/feast/pyspark/launcher.py | 29 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py index a8980f6a5c8..d876fa03e8d 100644 --- a/sdk/python/feast/data_format.py +++ b/sdk/python/feast/data_format.py @@ -8,7 +8,7 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# WITHOUT WARRANTIES OR CONDITIONS OF ANY aIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @@ -21,7 +21,7 @@ class FileFormat(ABC): """ - Defines an abtract file forma used to encode feature data in filesf + Defines an abtract file forma used to encode feature data in files """ @abstractmethod @@ -45,6 +45,12 @@ def from_proto(cls, proto): return ParquetFormat() raise NotImplementedError(f"FileFormat is unsupported: {fmt}") + def __str__(self): + """ + String representation of the file format passed to spark + """ + raise NotImplementedError() + class ParquetFormat(FileFormat): """ @@ -54,6 +60,9 @@ class ParquetFormat(FileFormat): def to_proto(self): return FileFormatProto(parquet_format=FileFormatProto.ParquetFormat()) + def __str__(self): + return "parquet" + class StreamFormat(ABC): """ diff --git a/sdk/python/feast/pyspark/launcher.py b/sdk/python/feast/pyspark/launcher.py index 61bf16cef62..9f1c40ab2a6 100644 --- a/sdk/python/feast/pyspark/launcher.py +++ b/sdk/python/feast/pyspark/launcher.py @@ -1,7 +1,7 @@ import shutil import tempfile from datetime import datetime -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, List, Union, cast from urllib.parse import urlparse from feast.config import Config @@ -86,8 +86,8 @@ def resolve_launcher(config: Config) -> JobLauncher: _SOURCES = { - FileSource: ("file", "file_options", {"path": "file_url", "format": "file_format"}), - BigQuerySource: ("bq", "bigquery_options", {"table_ref": "table_ref"}), + FileSource: ("file", "file_options"), + BigQuerySource: ("bq", "bigquery_options"), } @@ -99,17 +99,18 @@ def _source_to_argument(source: DataSource): "date_partition_column": source.date_partition_column, } - kind, option_field, extra_properties = _SOURCES[type(source)] - - properties = { - **common_properties, - **{ - k: getattr(getattr(source, option_field), ref) - for k, ref in extra_properties.items() - }, - } - - return {kind: properties} + kind, option_field = _SOURCES[type(source)] + properties = {**common_properties} + if type(source) == FileSource: + file_source = cast(FileSource, source) + properties["path"] = file_source.file_options.file_url + properties["format"] = str(file_source.file_options.file_format) + return {kind: properties} + if type(source) == BigQuerySource: + bq_source = cast(BigQuerySource, source) + properties["table_ref"] = bq_source.bigquery_options.table_ref + return {kind: properties} + raise NotImplementedError(f"Unsupported Datasource: {type(source)}") def _feature_table_to_argument(client: "Client", feature_table: FeatureTable):