Skip to content

feat: Write logged features to an offline store (Python API) #2574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 16 commits into from
Apr 26, 2022
Merged

feat: Write logged features to an offline store (Python API) #2574

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
2043767
write logs to offline store
pyalex Apr 19, 2022
f3f9998
format
pyalex Apr 19, 2022
af58966
fix after rebase
pyalex Apr 19, 2022
2681b3c
fix tests
pyalex Apr 19, 2022
5637da6
handle table not found in tests
pyalex Apr 19, 2022
43c97c2
some api docs
pyalex Apr 20, 2022
c5c7fbe
fix import
pyalex Apr 21, 2022
eaea5dd
use predefined schema in tests
pyalex Apr 21, 2022
7ac0b83
address pr comments
pyalex Apr 21, 2022
40fe028
more api docs
pyalex Apr 21, 2022
65a1a77
add proto attr to snowflake dest
pyalex Apr 21, 2022
4b983d6
add prefixes to system fields
pyalex Apr 22, 2022
0fc3e5a
add custom destination
pyalex Apr 22, 2022
9d1eb1a
move partition columns to destination config
pyalex Apr 26, 2022
ede26b2
after rebase
pyalex Apr 26, 2022
b2ef41f
allow data source creator implementations w/o logging destination
pyalex Apr 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add custom destination 
Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
  • Loading branch information
@pyalex
pyalex committed Apr 26, 2022
commit 0fc3e5a316134e814be6f2fb99bc7dbd14e33d60
6 changes: 6 additions & 0 deletions protos/feast/core/FeatureService.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ message LoggingConfig {
BigQueryDestination bigquery_destination = 4;
RedshiftDestination redshift_destination = 5;
SnowflakeDestination snowflake_destination = 6;
CustomDestination custom_destination = 7;
}

message FileDestination {
Expand All @@ -80,4 +81,9 @@ message LoggingConfig {
message SnowflakeDestination {
string table = 1;
}

message CustomDestination {
string kind = 1;
map<string, string> config = 2;
}
}
15 changes: 8 additions & 7 deletions sdk/python/feast/feature_logging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,10 @@ class LoggingDestination:
It is implementation specific - each offline store must implement LoggingDestination subclass.

Kind of logging destination will be determined by matching attribute name in LoggingConfig protobuf message
and "_proto_attr_name" property of each subclass.
and "_proto_kind" property of each subclass.
"""

_proto_attr_name: str
_proto_kind: str

@classmethod
@abc.abstractmethod
Expand All @@ -158,13 +158,14 @@ def __init__(self, destination: LoggingDestination):

@classmethod
def from_proto(cls, config_proto: LoggingConfigProto) -> Optional["LoggingConfig"]:
proto_attr_name = cast(str, config_proto.WhichOneof("destination"))
if proto_attr_name is None:
proto_kind = cast(str, config_proto.WhichOneof("destination"))
if proto_kind is None:
return

destination_class = _DestinationRegistry.classes_by_proto_attr_name[
proto_attr_name
]
if proto_kind == "custom_destination":
proto_kind = config_proto.custom_destination.kind

destination_class = _DestinationRegistry.classes_by_proto_attr_name[proto_kind]
return LoggingConfig(destination=destination_class.from_proto(config_proto))

def to_proto(self) -> LoggingConfigProto:
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/offline_stores/bigquery_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ def to_data_source(self) -> DataSource:


class BigQueryLoggingDestination(LoggingDestination):
_proto_attr_name = "bigquery_destination"
_proto_kind = "bigquery_destination"

table: str

Expand Down
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/offline_stores/file_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ def to_data_source(self) -> DataSource:


class FileLoggingDestination(LoggingDestination):
_proto_attr_name = "file_destination"
_proto_kind = "file_destination"

path: str
s3_endpoint_override: str
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/offline_stores/redshift_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ def to_data_source(self) -> DataSource:


class RedshiftLoggingDestination(LoggingDestination):
_proto_attr_name = "redshift_destination"
_proto_kind = "redshift_destination"

table: str

Expand Down
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/offline_stores/snowflake_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ def to_data_source(self) -> DataSource:
class SnowflakeLoggingDestination(LoggingDestination):
table: str

_proto_attr_name = "snowflake_destination"
_proto_kind = "snowflake_destination"

def __init__(self, table: str):
self.table = table
Expand Down