Skip to content

Add methods to offline/online stores to specify supported upstream sources #1662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
achals wants to merge 2 commits into from
Closed

Add methods to offline/online stores to specify supported upstream sources #1662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc2cb77
Add methods to offline/online stores to specify supported upstream so…
achals Jun 23, 2021
0635221
make format
achals Jun 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions sdk/python/feast/infra/offline_stores/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
_get_requested_feature_views_to_features_dict,
)
from feast.registry import Registry
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.repo_config import FeastOfflineStoreConfigBaseModel, RepoConfig

try:
from google.api_core.exceptions import NotFound
Expand All @@ -36,7 +36,7 @@
raise FeastExtrasDependencyImportError("gcp", str(e))


class BigQueryOfflineStoreConfig(FeastConfigBaseModel):
class BigQueryOfflineStoreConfig(FeastOfflineStoreConfigBaseModel):
""" Offline store config for GCP BigQuery """

type: Literal["bigquery"] = "bigquery"
Expand All @@ -48,6 +48,9 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel):
project_id: Optional[StrictStr] = None
""" (optional) GCP project name used for the BigQuery offline store """

def supports_data_source(self, data_source: DataSource) -> bool:
return isinstance(data_source, BigQuerySource)


class BigQueryOfflineStore(OfflineStore):
@staticmethod
Expand Down
7 changes: 5 additions & 2 deletions sdk/python/feast/infra/offline_stores/file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@
_run_field_mapping,
)
from feast.registry import Registry
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.repo_config import FeastOfflineStoreConfigBaseModel, RepoConfig


class FileOfflineStoreConfig(FeastConfigBaseModel):
class FileOfflineStoreConfig(FeastOfflineStoreConfigBaseModel):
""" Offline store config for local (file-based) store """

type: Literal["file"] = "file"
""" Offline store type selector"""

def supports_data_source(self, data_source: DataSource) -> bool:
return isinstance(data_source, FileSource)


class FileRetrievalJob(RetrievalJob):
def __init__(self, evaluation_function: Callable):
Expand Down
8 changes: 6 additions & 2 deletions sdk/python/feast/infra/online_stores/datastore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from feast.infra.online_stores.online_store import OnlineStore
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
from feast.protos.feast.types.Value_pb2 import Value as ValueProto
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.repo_config import FeastOnlineStoreConfigBaseModel, RepoConfig

try:
from google.auth.exceptions import DefaultCredentialsError
Expand All @@ -42,7 +42,7 @@
]


class DatastoreOnlineStoreConfig(FeastConfigBaseModel):
class DatastoreOnlineStoreConfig(FeastOnlineStoreConfigBaseModel):
""" Online store config for GCP Datastore """

type: Literal["datastore"] = "datastore"
Expand All @@ -60,6 +60,10 @@ class DatastoreOnlineStoreConfig(FeastConfigBaseModel):
write_batch_size: Optional[PositiveInt] = 50
""" (optional) Amount of feature rows per batch being written into Datastore"""

def supports_offline_store(self, offline_store: Any) -> bool:
# We're defaulting to supporting all offline stores for now.
return offline_store.type == "bigquery"


class DatastoreOnlineStore(OnlineStore):
"""
Expand Down
8 changes: 6 additions & 2 deletions sdk/python/feast/infra/online_stores/redis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from feast.infra.online_stores.online_store import OnlineStore
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
from feast.protos.feast.types.Value_pb2 import Value as ValueProto
from feast.repo_config import FeastConfigBaseModel
from feast.repo_config import FeastOnlineStoreConfigBaseModel

try:
from redis import Redis
Expand All @@ -43,7 +43,7 @@ class RedisType(str, Enum):
redis_cluster = "redis_cluster"


class RedisOnlineStoreConfig(FeastConfigBaseModel):
class RedisOnlineStoreConfig(FeastOnlineStoreConfigBaseModel):
"""Online store config for Redis store"""

type: Literal["redis"] = "redis"
Expand All @@ -56,6 +56,10 @@ class RedisOnlineStoreConfig(FeastConfigBaseModel):
"""Connection string containing the host, port, and configuration parameters for Redis
format: host:port,parameter1,parameter2 eg. redis:6379,db=0 """

def supports_offline_store(self, offline_store: Any) -> bool:
# We're defaulting to supporting all offline stores for now.
return True


class RedisOnlineStore(OnlineStore):
_client: Optional[Union[Redis, RedisCluster]] = None
Expand Down
7 changes: 5 additions & 2 deletions sdk/python/feast/infra/online_stores/sqlite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@
from feast.infra.online_stores.online_store import OnlineStore
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
from feast.protos.feast.types.Value_pb2 import Value as ValueProto
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.repo_config import FeastOnlineStoreConfigBaseModel, RepoConfig


class SqliteOnlineStoreConfig(FeastConfigBaseModel):
class SqliteOnlineStoreConfig(FeastOnlineStoreConfigBaseModel):
""" Online store config for local (SQLite-based) store """

type: Literal[
Expand All @@ -42,6 +42,9 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel):
path: StrictStr = "data/online.db"
""" (optional) Path to sqlite db """

def supports_offline_store(self, offline_store: Any) -> bool:
return offline_store.type == "file"


class SqliteOnlineStore(OnlineStore):
"""
Expand Down
20 changes: 19 additions & 1 deletion sdk/python/feast/repo_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from abc import abstractmethod
from pathlib import Path
from typing import Any

Expand All @@ -6,6 +7,7 @@
from pydantic.error_wrappers import ErrorWrapper
from pydantic.typing import Dict, Optional, Union

from feast.data_source import DataSource
from feast.importer import get_class_from_type
from feast.usage import log_exceptions

Expand All @@ -32,13 +34,29 @@ class Config:
extra = "allow"


class FeastConfigBaseModel(BaseModel):
class FeastOfflineStoreConfigBaseModel(BaseModel):
""" Feast Pydantic Configuration Class """

class Config:
arbitrary_types_allowed = True
extra = "forbid"

@abstractmethod
def supports_data_source(self, data_source: DataSource) -> bool:
...


class FeastOnlineStoreConfigBaseModel(BaseModel):
""" Feast Pydantic Configuration Class """

class Config:
arbitrary_types_allowed = True
extra = "forbid"

@abstractmethod
def supports_offline_store(self, offline_store: Any) -> bool:
...

Comment on lines +56 to +59
Copy link
Copy Markdown
Collaborator

@tsotnet tsotnet Jun 23, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does online store need to define which offline stores to support? The online store and offline store do no interact with each other directly.

I'd say let providers define which online store & offline store they support.

Copy link
Copy Markdown
Member Author

@achals achals Jun 23, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does online store need to define which offline stores to support? The online store and offline store do no interact with each other directly.

I'd say let providers define which online store & offline store they support.

Spoke offline - I'm going to scrap this PR because there's no explicitly unsupported combination at the moment. We should wait until the need for this kind of API emerges, possibly after we have an AWS provider.


class RegistryConfig(FeastBaseModel):
""" Metadata Store Configuration. Configuration that relates to reading from and writing to the Feast registry."""
Expand Down
3 changes: 3 additions & 0 deletions sdk/python/feast/repo_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,11 @@ def apply_total(repo_config: RepoConfig, repo_path: Path):

# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
assert repo_config.offline_store.supports_data_source(data_source)
data_source.validate()

assert repo_config.online_store.supports_offline_store(repo_config.offline_store)

update_data_sources_with_inferred_event_timestamp_col(data_sources)
for view in repo.feature_views:
view.infer_features_from_input_source()
Expand Down