Skip to content

Commit b4d12bd

Browse files
felixwang9817felixwang9817
authored
Refactor all importer logic to belong in feast.importer (feast-dev#2199)
* Refactor all importer logic to belong in feast.importer Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix unit tests error messages to match feast.errors Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Rename get_class_from_module to import_class Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Change class_type checking logic Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Change error name to FeastInvalidBaseClass Signed-off-by: Felix Wang <wangfelix98@gmail.com>
1 parent f969e53 commit b4d12bd

File tree

9 files changed

+63
-81
lines changed

9 files changed

+63
-81
lines changed

‎sdk/python/feast/errors.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -103,14 +103,16 @@ def __init__(self, feature_server_type: str):
103103

104104

105105
class FeastModuleImportError(Exception):
106-
def __init__(self, module_name: str, module_type: str):
107-
super().__init__(f"Could not import {module_type} module '{module_name}'")
106+
def __init__(self, module_name: str, class_name: str):
107+
super().__init__(
108+
f"Could not import module '{module_name}' while attempting to load class '{class_name}'"
109+
)
108110

109111

110112
class FeastClassImportError(Exception):
111-
def __init__(self, module_name, class_name, class_type="provider"):
113+
def __init__(self, module_name: str, class_name: str):
112114
super().__init__(
113-
f"Could not import {class_type} '{class_name}' from module '{module_name}'"
115+
f"Could not import class '{class_name}' from module '{module_name}'"
114116
)
115117

116118

@@ -168,11 +170,10 @@ def __init__(self, online_store_class_name: str):
168170
)
169171

170172

171-
class FeastClassInvalidName(Exception):
173+
class FeastInvalidBaseClass(Exception):
172174
def __init__(self, class_name: str, class_type: str):
173175
super().__init__(
174-
f"Config Class '{class_name}' "
175-
f"should end with the string `{class_type}`.'"
176+
f"Class '{class_name}' should have `{class_type}` as a base class."
176177
)
177178

178179

‎sdk/python/feast/importer.py‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,47 @@
11
import importlib
22

3-
from feast import errors
3+
from feast.errors import (
4+
FeastClassImportError,
5+
FeastInvalidBaseClass,
6+
FeastModuleImportError,
7+
)
48

59

6-
def get_class_from_type(module_name: str, class_name: str, class_type: str):
7-
if not class_name.endswith(class_type):
8-
raise errors.FeastClassInvalidName(class_name, class_type)
10+
def import_class(module_name: str, class_name: str, class_type: str = None):
11+
"""
12+
Dynamically loads and returns a class from a module.
913
10-
# Try importing the module that contains the custom provider
14+
Args:
15+
module_name: The name of the module.
16+
class_name: The name of the class.
17+
class_type: Optional name of a base class of the class.
18+
19+
Raises:
20+
FeastInvalidBaseClass: If the class name does not end with the specified suffix.
21+
FeastModuleImportError: If the module cannot be imported.
22+
FeastClassImportError: If the class cannot be imported.
23+
"""
24+
# Try importing the module.
1125
try:
1226
module = importlib.import_module(module_name)
1327
except Exception as e:
1428
# The original exception can be anything - either module not found,
1529
# or any other kind of error happening during the module import time.
1630
# So we should include the original error as well in the stack trace.
17-
raise errors.FeastModuleImportError(module_name, class_type) from e
31+
raise FeastModuleImportError(module_name, class_name) from e
1832

19-
# Try getting the provider class definition
33+
# Try getting the class.
2034
try:
2135
_class = getattr(module, class_name)
2236
except AttributeError:
2337
# This can only be one type of error, when class_name attribute does not exist in the module
2438
# So we don't have to include the original exception here
25-
raise errors.FeastClassImportError(
26-
module_name, class_name, class_type=class_type
27-
) from None
39+
raise FeastClassImportError(module_name, class_name) from None
40+
41+
# Check if the class is a subclass of the base class.
42+
if class_type and not any(
43+
base_class.__name__ == class_type for base_class in _class.mro()
44+
):
45+
raise FeastInvalidBaseClass(class_name, class_type)
46+
2847
return _class

‎sdk/python/feast/infra/infra_object.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from dataclasses import dataclass, field
1616
from typing import Any, List
1717

18-
from feast.importer import get_class_from_type
18+
from feast.importer import import_class
1919
from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto
2020
from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto
2121

@@ -106,4 +106,4 @@ def from_proto(cls, infra_proto: InfraProto):
106106

107107
def _get_infra_object_class_from_type(infra_object_class_type: str):
108108
module_name, infra_object_class_name = infra_object_class_type.rsplit(".", 1)
109-
return get_class_from_type(module_name, infra_object_class_name, "Object")
109+
return import_class(module_name, infra_object_class_name)

‎sdk/python/feast/infra/offline_stores/offline_utils.py‎

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import importlib
21
import uuid
32
from dataclasses import asdict, dataclass
43
from datetime import datetime, timedelta
@@ -12,11 +11,10 @@
1211
import feast
1312
from feast.errors import (
1413
EntityTimestampInferenceException,
15-
FeastClassImportError,
1614
FeastEntityDFMissingColumnsError,
17-
FeastModuleImportError,
1815
)
1916
from feast.feature_view import FeatureView
17+
from feast.importer import import_class
2018
from feast.infra.offline_stores.offline_store import OfflineStore
2119
from feast.infra.provider import _get_requested_feature_views_to_features_dict
2220
from feast.registry import Registry
@@ -204,27 +202,10 @@ def get_temp_entity_table_name() -> str:
204202
return "feast_entity_df_" + uuid.uuid4().hex
205203

206204

207-
def get_offline_store_from_config(offline_store_config: Any,) -> OfflineStore:
208-
"""Get the offline store from offline store config"""
209-
205+
def get_offline_store_from_config(offline_store_config: Any) -> OfflineStore:
206+
"""Creates an offline store corresponding to the given offline store config."""
210207
module_name = offline_store_config.__module__
211208
qualified_name = type(offline_store_config).__name__
212-
store_class_name = qualified_name.replace("Config", "")
213-
try:
214-
module = importlib.import_module(module_name)
215-
except Exception as e:
216-
# The original exception can be anything - either module not found,
217-
# or any other kind of error happening during the module import time.
218-
# So we should include the original error as well in the stack trace.
219-
raise FeastModuleImportError(module_name, "OfflineStore") from e
220-
221-
# Try getting the provider class definition
222-
try:
223-
offline_store_class = getattr(module, store_class_name)
224-
except AttributeError:
225-
# This can only be one type of error, when class_name attribute does not exist in the module
226-
# So we don't have to include the original exception here
227-
raise FeastClassImportError(
228-
module_name, store_class_name, class_type="OfflineStore"
229-
) from None
209+
class_name = qualified_name.replace("Config", "")
210+
offline_store_class = import_class(module_name, class_name, "OfflineStore")
230211
return offline_store_class()

‎sdk/python/feast/infra/online_stores/helpers.py‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
import importlib
21
import struct
32
from typing import Any, List
43

54
import mmh3
65

7-
from feast import errors
6+
from feast.importer import import_class
87
from feast.infra.key_encoding_utils import (
98
serialize_entity_key,
109
serialize_entity_key_prefix,
@@ -13,29 +12,12 @@
1312
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
1413

1514

16-
def get_online_store_from_config(online_store_config: Any,) -> OnlineStore:
17-
"""Get the online store from online store config"""
18-
15+
def get_online_store_from_config(online_store_config: Any) -> OnlineStore:
16+
"""Creates an online store corresponding to the given online store config."""
1917
module_name = online_store_config.__module__
2018
qualified_name = type(online_store_config).__name__
21-
store_class_name = qualified_name.replace("Config", "")
22-
try:
23-
module = importlib.import_module(module_name)
24-
except Exception as e:
25-
# The original exception can be anything - either module not found,
26-
# or any other kind of error happening during the module import time.
27-
# So we should include the original error as well in the stack trace.
28-
raise errors.FeastModuleImportError(module_name, "OnlineStore") from e
29-
30-
# Try getting the provider class definition
31-
try:
32-
online_store_class = getattr(module, store_class_name)
33-
except AttributeError:
34-
# This can only be one type of error, when class_name attribute does not exist in the module
35-
# So we don't have to include the original exception here
36-
raise errors.FeastClassImportError(
37-
module_name, store_class_name, class_type="OnlineStore"
38-
) from None
19+
class_name = qualified_name.replace("Config", "")
20+
online_store_class = import_class(module_name, class_name, "OnlineStore")
3921
return online_store_class()
4022

4123

‎sdk/python/feast/infra/provider.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
import pyarrow
99
from tqdm import tqdm
1010

11-
from feast import errors, importer
11+
from feast import errors
1212
from feast.entity import Entity
1313
from feast.feature_view import DUMMY_ENTITY_ID, FeatureView
14+
from feast.importer import import_class
1415
from feast.infra.offline_stores.offline_store import RetrievalJob
1516
from feast.on_demand_feature_view import OnDemandFeatureView
1617
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
@@ -172,7 +173,7 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider:
172173
# For example, provider 'foo.bar.MyProvider' will be parsed into 'foo.bar' and 'MyProvider'
173174
module_name, class_name = provider.rsplit(".", 1)
174175

175-
cls = importer.get_class_from_type(module_name, class_name, "Provider")
176+
cls = import_class(module_name, class_name, "Provider")
176177

177178
return cls(config)
178179

‎sdk/python/feast/registry.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
from google.protobuf.json_format import MessageToDict
2424
from proto import Message
2525

26-
from feast import importer
2726
from feast.base_feature_view import BaseFeatureView
2827
from feast.diff.FcoDiff import (
2928
FcoDiff,
@@ -42,6 +41,7 @@
4241
)
4342
from feast.feature_service import FeatureService
4443
from feast.feature_view import FeatureView
44+
from feast.importer import import_class
4545
from feast.infra.infra_object import Infra
4646
from feast.on_demand_feature_view import OnDemandFeatureView
4747
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
@@ -75,9 +75,7 @@ def get_registry_store_class_from_type(registry_store_type: str):
7575
registry_store_type = REGISTRY_STORE_CLASS_FOR_TYPE[registry_store_type]
7676
module_name, registry_store_class_name = registry_store_type.rsplit(".", 1)
7777

78-
return importer.get_class_from_type(
79-
module_name, registry_store_class_name, "RegistryStore"
80-
)
78+
return import_class(module_name, registry_store_class_name, "RegistryStore")
8179

8280

8381
def get_registry_store_class_from_scheme(registry_path: str):

‎sdk/python/feast/repo_config.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
FeastFeatureServerTypeSetError,
2121
FeastProviderNotSetError,
2222
)
23-
from feast.importer import get_class_from_type
23+
from feast.importer import import_class
2424
from feast.usage import log_exceptions
2525

2626
# These dict exists so that:
@@ -302,7 +302,7 @@ def __repr__(self) -> str:
302302

303303
def get_data_source_class_from_type(data_source_type: str):
304304
module_name, config_class_name = data_source_type.rsplit(".", 1)
305-
return get_class_from_type(module_name, config_class_name, "Source")
305+
return import_class(module_name, config_class_name, "DataSource")
306306

307307

308308
def get_online_config_from_type(online_store_type: str):
@@ -313,7 +313,7 @@ def get_online_config_from_type(online_store_type: str):
313313
module_name, online_store_class_type = online_store_type.rsplit(".", 1)
314314
config_class_name = f"{online_store_class_type}Config"
315315

316-
return get_class_from_type(module_name, config_class_name, config_class_name)
316+
return import_class(module_name, config_class_name, config_class_name)
317317

318318

319319
def get_offline_config_from_type(offline_store_type: str):
@@ -324,7 +324,7 @@ def get_offline_config_from_type(offline_store_type: str):
324324
module_name, offline_store_class_type = offline_store_type.rsplit(".", 1)
325325
config_class_name = f"{offline_store_class_type}Config"
326326

327-
return get_class_from_type(module_name, config_class_name, config_class_name)
327+
return import_class(module_name, config_class_name, config_class_name)
328328

329329

330330
def get_feature_server_config_from_type(feature_server_type: str):
@@ -334,7 +334,7 @@ def get_feature_server_config_from_type(feature_server_type: str):
334334

335335
feature_server_type = FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE[feature_server_type]
336336
module_name, config_class_name = feature_server_type.rsplit(".", 1)
337-
return get_class_from_type(module_name, config_class_name, config_class_name)
337+
return import_class(module_name, config_class_name, config_class_name)
338338

339339

340340
def load_repo_config(repo_path: Path) -> RepoConfig:

‎sdk/python/tests/integration/registration/test_cli.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,14 +211,14 @@ def test_3rd_party_providers() -> None:
211211
return_code, output = runner.run_with_output(["apply"], cwd=repo_path)
212212
assertpy.assert_that(return_code).is_equal_to(1)
213213
assertpy.assert_that(output).contains(
214-
b"Could not import Provider module 'feast_foo'"
214+
b"Could not import module 'feast_foo' while attempting to load class 'Provider'"
215215
)
216216
# Check with incorrect third-party provider name (with dots)
217217
with setup_third_party_provider_repo("foo.FooProvider") as repo_path:
218218
return_code, output = runner.run_with_output(["apply"], cwd=repo_path)
219219
assertpy.assert_that(return_code).is_equal_to(1)
220220
assertpy.assert_that(output).contains(
221-
b"Could not import Provider 'FooProvider' from module 'foo'"
221+
b"Could not import class 'FooProvider' from module 'foo'"
222222
)
223223
# Check with correct third-party provider name
224224
with setup_third_party_provider_repo("foo.provider.FooProvider") as repo_path:
@@ -243,14 +243,14 @@ def test_3rd_party_registry_store() -> None:
243243
return_code, output = runner.run_with_output(["apply"], cwd=repo_path)
244244
assertpy.assert_that(return_code).is_equal_to(1)
245245
assertpy.assert_that(output).contains(
246-
b"Could not import RegistryStore module 'feast_foo'"
246+
b"Could not import module 'feast_foo' while attempting to load class 'RegistryStore'"
247247
)
248248
# Check with incorrect third-party registry store name (with dots)
249249
with setup_third_party_registry_store_repo("foo.FooRegistryStore") as repo_path:
250250
return_code, output = runner.run_with_output(["apply"], cwd=repo_path)
251251
assertpy.assert_that(return_code).is_equal_to(1)
252252
assertpy.assert_that(output).contains(
253-
b"Could not import RegistryStore 'FooRegistryStore' from module 'foo'"
253+
b"Could not import class 'FooRegistryStore' from module 'foo'"
254254
)
255255
# Check with correct third-party registry store name
256256
with setup_third_party_registry_store_repo(

0 commit comments

Comments
 (0)