Skip to content

Commit 3553737

Browse files
woopwoop
authored
Add support for Pydantic as configuration loader (feast-dev#1432)
* Add support for Pydantic as configuration loader Signed-off-by: Willem Pienaar <git@willem.co> * Add sphinx sources back Signed-off-by: Willem Pienaar <git@willem.co>
1 parent 0f0d83e commit 3553737

6 files changed

Lines changed: 58 additions & 30 deletions

File tree

‎sdk/python/docs/source/feast.rst‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,14 @@ feast.data\_source module
6363
:undoc-members:
6464
:show-inheritance:
6565

66+
feast.driver\_test\_data module
67+
-------------------------------
68+
69+
.. automodule:: feast.driver_test_data
70+
:members:
71+
:undoc-members:
72+
:show-inheritance:
73+
6674
feast.entity module
6775
-------------------
6876

@@ -71,6 +79,14 @@ feast.entity module
7179
:undoc-members:
7280
:show-inheritance:
7381

82+
feast.example\_repo module
83+
--------------------------
84+
85+
.. automodule:: feast.example_repo
86+
:members:
87+
:undoc-members:
88+
:show-inheritance:
89+
7490
feast.feature module
7591
--------------------
7692

‎sdk/python/feast/feature_store.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def __init__(
6868
project="default",
6969
provider="local",
7070
online_store=OnlineStoreConfig(
71-
local=LocalOnlineStoreConfig("online_store.db")
71+
local=LocalOnlineStoreConfig(path="online_store.db")
7272
),
7373
)
7474

‎sdk/python/feast/repo_config.py‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,55 @@
11
from pathlib import Path
2-
from typing import NamedTuple, Optional
2+
from typing import Optional
33

44
import yaml
5-
from bindr import bind
6-
from jsonschema import ValidationError, validate
5+
from pydantic import BaseModel, StrictStr, ValidationError
76

87

9-
class LocalOnlineStoreConfig(NamedTuple):
8+
class FeastBaseModel(BaseModel):
9+
""" Feast Pydantic Configuration Class """
10+
11+
class Config:
12+
arbitrary_types_allowed = True
13+
extra = "forbid"
14+
15+
16+
class LocalOnlineStoreConfig(FeastBaseModel):
1017
""" Online store config for local (SQLite-based) online store """
1118

12-
path: str
19+
path: StrictStr
1320
""" str: Path to sqlite db """
1421

1522

16-
class DatastoreOnlineStoreConfig(NamedTuple):
23+
class DatastoreOnlineStoreConfig(FeastBaseModel):
1724
""" Online store config for GCP Datastore """
1825

19-
project_id: str
26+
project_id: StrictStr
2027
""" str: GCP Project Id """
2128

2229

23-
class OnlineStoreConfig(NamedTuple):
30+
class OnlineStoreConfig(FeastBaseModel):
2431
datastore: Optional[DatastoreOnlineStoreConfig] = None
2532
""" DatastoreOnlineStoreConfig: Optional DatastoreConfig """
33+
2634
local: Optional[LocalOnlineStoreConfig] = None
2735
""" LocalOnlineStoreConfig: Optional local online store config """
2836

2937

30-
class RepoConfig(NamedTuple):
38+
class RepoConfig(FeastBaseModel):
3139
""" Repo config. Typically loaded from `feature_store.yaml` """
3240

33-
metadata_store: str
41+
metadata_store: StrictStr
3442
""" str: Path to metadata store. Can be a local path, or remote object storage path, e.g. gcs://foo/bar """
35-
project: str
43+
44+
project: StrictStr
3645
""" str: Feast project id. This can be any alphanumeric string up to 16 characters.
3746
You can have multiple independent feature repositories deployed to the same cloud
3847
provider account, as long as they have different project ids.
3948
"""
40-
provider: str
49+
50+
provider: StrictStr
4151
""" str: local or gcp """
52+
4253
online_store: Optional[OnlineStoreConfig] = None
4354
""" OnlineStoreConfig: Online store configuration (optional depending on provider) """
4455

@@ -79,20 +90,18 @@ class RepoConfig(NamedTuple):
7990

8091

8192
class FeastConfigError(Exception):
82-
def __init__(self, error_message, error_path, config_path):
93+
def __init__(self, error_message, config_path):
8394
self._error_message = error_message
84-
self._error_path = error_path
8595
self._config_path = config_path
8696
super().__init__(self._error_message)
8797

8898
def __str__(self) -> str:
89-
if self._error_path:
90-
return f'{self._error_message} under {"->".join(self._error_path)} in {self._config_path}'
91-
else:
92-
return f"{self._error_message} in {self._config_path}"
99+
return f"{self._error_message}\nat {self._config_path}"
93100

94101
def __repr__(self) -> str:
95-
return f"FeastConfigError({repr(self._error_message)}, {repr(self._error_path)}, {repr(self._config_path)})"
102+
return (
103+
f"FeastConfigError({repr(self._error_message)}, {repr(self._config_path)})"
104+
)
96105

97106

98107
def load_repo_config(repo_path: Path) -> RepoConfig:
@@ -101,7 +110,6 @@ def load_repo_config(repo_path: Path) -> RepoConfig:
101110
with open(config_path) as f:
102111
raw_config = yaml.safe_load(f)
103112
try:
104-
validate(raw_config, config_schema)
105-
return bind(RepoConfig, raw_config)
113+
return RepoConfig(**raw_config)
106114
except ValidationError as e:
107-
raise FeastConfigError(e.message, e.absolute_path, config_path)
115+
raise FeastConfigError(e, config_path)

‎sdk/python/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939

4040
REQUIRED = [
4141
"Click==7.*",
42-
"bindr",
4342
"fastavro>=0.22.11,<0.23",
4443
"google-api-core>=1.23.0",
4544
"google-cloud-bigquery>=2.0.*",
@@ -56,6 +55,7 @@
5655
"pandavro==1.5.*",
5756
"protobuf>=3.10",
5857
"pyarrow==2.0.0",
58+
"pydantic>=1.0.0",
5959
"PyYAML==5.3.*",
6060
"tabulate==0.8.*",
6161
"toml==0.10.*",

‎sdk/python/tests/test_historical_retrieval.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ def test_historical_features_from_parquet_sources():
234234
provider="local",
235235
online_store=OnlineStoreConfig(
236236
local=LocalOnlineStoreConfig(
237-
os.path.join(temp_dir, "online_store.db"),
237+
path=os.path.join(temp_dir, "online_store.db")
238238
)
239239
),
240240
)

‎sdk/python/tests/test_repo_config.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import re
21
import tempfile
32
from pathlib import Path
43
from textwrap import dedent
@@ -26,7 +25,7 @@ def _test_config(self, config_text, expect_error: Optional[str]):
2625
error = e
2726

2827
if expect_error is not None:
29-
assert re.search(expect_error, str(error)) is not None
28+
assert expect_error in str(error)
3029
else:
3130
assert error is None
3231

@@ -69,7 +68,8 @@ def test_errors(self) -> None:
6968
path: "online_store.db"
7069
"""
7170
),
72-
expect_error=r"'that_field_should_not_be_here' was unexpected.*online_store->local",
71+
expect_error="online_store -> local -> that_field_should_not_be_here\n"
72+
" extra fields not permitted (type=value_error.extra)",
7373
)
7474

7575
self._test_config(
@@ -83,7 +83,9 @@ def test_errors(self) -> None:
8383
path: 100500
8484
"""
8585
),
86-
expect_error=r"100500 is not of type 'string'",
86+
expect_error="1 validation error for RepoConfig\n"
87+
"online_store -> local -> path\n"
88+
" str type expected (type=type_error.str)",
8789
)
8890

8991
self._test_config(
@@ -96,5 +98,7 @@ def test_errors(self) -> None:
9698
path: foo
9799
"""
98100
),
99-
expect_error=r"'project' is a required property",
101+
expect_error="1 validation error for RepoConfig\n"
102+
"project\n"
103+
" field required (type=value_error.missing)",
100104
)

0 commit comments

Comments
 (0)