Skip to content

Commit 693e9df

Browse files
toping4445younggyu-oh
toping4445
authored and
younggyu-oh
committed
fixed bugs, cleaned code, added AthenaDataSourceCreator
Signed-off-by: Youngkyu OH <toping4445@gmail.com>
1 parent 63d541d commit 693e9df

16 files changed

Lines changed: 1688 additions & 2 deletions

File tree

‎protos/feast/core/DataSource.proto‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ message DataSource {
4949
PUSH_SOURCE = 9;
5050
BATCH_TRINO = 10;
5151
BATCH_SPARK = 11;
52+
BATCH_ATHENA = 12;
5253
}
5354

5455
// Unique name of data source within the project
@@ -171,6 +172,22 @@ message DataSource {
171172
string database = 4;
172173
}
173174

175+
// Defines options for DataSource that sources features from a Athena Query
176+
message AthenaOptions {
177+
// Athena table name
178+
string table = 1;
179+
180+
// SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
181+
// entity columns
182+
string query = 2;
183+
184+
// Athena database name
185+
string database = 3;
186+
187+
// Athena schema name
188+
string data_source = 4;
189+
}
190+
174191
// Defines options for DataSource that sources features from a Snowflake Query
175192
message SnowflakeOptions {
176193
// Snowflake table name
@@ -242,5 +259,6 @@ message DataSource {
242259
PushOptions push_options = 22;
243260
SparkOptions spark_options = 27;
244261
TrinoOptions trino_options = 30;
262+
AthenaOptions athena_options = 35;
245263
}
246264
}

‎protos/feast/core/FeatureService.proto‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ message LoggingConfig {
6060
RedshiftDestination redshift_destination = 5;
6161
SnowflakeDestination snowflake_destination = 6;
6262
CustomDestination custom_destination = 7;
63+
AthenaDestination athena_destination = 8;
6364
}
6465

6566
message FileDestination {
@@ -80,6 +81,11 @@ message LoggingConfig {
8081
string table_name = 1;
8182
}
8283

84+
message AthenaDestination {
85+
// Destination table name. data_source and database will be taken from an offline store config
86+
string table_name = 1;
87+
}
88+
8389
message SnowflakeDestination {
8490
// Destination table name. Schema and database will be taken from an offline store config
8591
string table_name = 1;

‎protos/feast/core/SavedDataset.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ message SavedDatasetStorage {
5959
DataSource.TrinoOptions trino_storage = 8;
6060
DataSource.SparkOptions spark_storage = 9;
6161
DataSource.CustomSourceOptions custom_storage = 10;
62+
DataSource.AthenaOptions athena_storage = 11;
6263
}
6364
}
6465

‎sdk/python/feast/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
from feast.infra.offline_stores.file_source import FileSource
99
from feast.infra.offline_stores.redshift_source import RedshiftSource
1010
from feast.infra.offline_stores.snowflake_source import SnowflakeSource
11+
from feast.infra.offline_stores.contrib.athena_offline_store.athena_source import (
12+
AthenaSource,
13+
)
1114

1215
from .batch_feature_view import BatchFeatureView
1316
from .data_source import KafkaSource, KinesisSource, PushSource, RequestSource
@@ -50,4 +53,5 @@
5053
"SnowflakeSource",
5154
"PushSource",
5255
"RequestSource",
56+
"AthenaSource",
5357
]

‎sdk/python/feast/data_source.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ def to_proto(self) -> DataSourceProto.KinesisOptions:
160160
DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource",
161161
DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestSource",
162162
DataSourceProto.SourceType.PUSH_SOURCE: "feast.data_source.PushSource",
163+
DataSourceProto.SourceType.BATCH_ATHENA: "feast.infra.offline_stores.contrib.athena_offline_store.athena_source.AthenaSource",
163164
}
164165

165166

‎sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)