Skip to content

Grouped inferencing statements together in apply methods for easier readability #1667

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 2 commits into from
Jun 25, 2021
Merged

Grouped inferencing statements together in apply methods for easier readability #1667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8efc9cd
grouped inferencing statements together
mavysavydav Jun 25, 2021
9b2c8db
update in testing
mavysavydav Jun 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions sdk/python/feast/feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
from feast.errors import FeastProviderLoginError, FeatureViewNotFoundException
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
update_data_sources_with_inferred_event_timestamp_col,
update_entities_with_inferred_types_from_feature_views,
)
from feast.infra.provider import Provider, RetrievalJob, get_provider
from feast.online_response import OnlineResponse, _infer_online_entity_rows
Expand Down Expand Up @@ -224,8 +224,11 @@ def apply(
assert isinstance(objects, list)

views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)]
entities_to_update = infer_entity_value_type_from_feature_views(
[ob for ob in objects if isinstance(ob, Entity)], views_to_update
entities_to_update = [ob for ob in objects if isinstance(ob, Entity)]

# Make inferences
update_entities_with_inferred_types_from_feature_views(
entities_to_update, views_to_update
)
update_data_sources_with_inferred_event_timestamp_col(
[view.input for view in views_to_update]
Expand Down
6 changes: 2 additions & 4 deletions sdk/python/feast/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
from feast.value_type import ValueType


def infer_entity_value_type_from_feature_views(
def update_entities_with_inferred_types_from_feature_views(
entities: List[Entity], feature_views: List[FeatureView]
) -> List[Entity]:
) -> None:
"""
Infer entity value type by examining schema of feature view input sources
"""
Expand Down Expand Up @@ -57,8 +57,6 @@ def infer_entity_value_type_from_feature_views(

entity.value_type = inferred_value_type

return entities


def update_data_sources_with_inferred_event_timestamp_col(
data_sources: List[Union[BigQuerySource, FileSource]],
Expand Down
30 changes: 13 additions & 17 deletions sdk/python/feast/repo_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
from feast import Entity, FeatureTable
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
update_data_sources_with_inferred_event_timestamp_col,
update_entities_with_inferred_types_from_feature_views,
)
from feast.infra.provider import get_provider
from feast.names import adjectives, animals
Expand Down Expand Up @@ -131,13 +131,19 @@ def apply_total(repo_config: RepoConfig, repo_path: Path):
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
repo = ParsedRepo(
feature_tables=repo.feature_tables,
entities=infer_entity_value_type_from_feature_views(
repo.entities, repo.feature_views
),
feature_views=repo.feature_views,
data_sources = [t.input for t in repo.feature_views]

# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate()

# Make inferences
update_entities_with_inferred_types_from_feature_views(
repo.entities, repo.feature_views
)
update_data_sources_with_inferred_event_timestamp_col(data_sources)
for view in repo.feature_views:
view.infer_features_from_input_source()

sys.dont_write_bytecode = False
for entity in repo.entities:
Expand All @@ -151,16 +157,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path):
for t in repo.feature_views:
repo_table_names.add(t.name)

data_sources = [t.input for t in repo.feature_views]

# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate()

update_data_sources_with_inferred_event_timestamp_col(data_sources)
for view in repo.feature_views:
view.infer_features_from_input_source()

tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
Expand Down
27 changes: 15 additions & 12 deletions sdk/python/tests/test_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@
from feast.errors import RegistryInferenceFailure
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
update_data_sources_with_inferred_event_timestamp_col,
update_entities_with_inferred_types_from_feature_views,
)


def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dataset_2):
def test_update_entities_with_inferred_types_from_feature_views(
simple_dataset_1, simple_dataset_2
):
with prep_file_source(
df=simple_dataset_1, event_timestamp_column="ts_1"
) as file_source, prep_file_source(
Expand All @@ -24,22 +26,23 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat
fv1 = FeatureView(name="fv1", entities=["id"], input=file_source, ttl=None,)
fv2 = FeatureView(name="fv2", entities=["id"], input=file_source_2, ttl=None,)

actual_1 = infer_entity_value_type_from_feature_views(
[Entity(name="id")], [fv1]
)
actual_2 = infer_entity_value_type_from_feature_views(
[Entity(name="id")], [fv2]
)
assert actual_1 == [Entity(name="id", value_type=ValueType.INT64)]
assert actual_2 == [Entity(name="id", value_type=ValueType.STRING)]
actual_1 = Entity(name="id")
actual_2 = Entity(name="id")

update_entities_with_inferred_types_from_feature_views([actual_1], [fv1])
update_entities_with_inferred_types_from_feature_views([actual_2], [fv2])
assert actual_1 == Entity(name="id", value_type=ValueType.INT64)
assert actual_2 == Entity(name="id", value_type=ValueType.STRING)

with pytest.raises(RegistryInferenceFailure):
# two viable data types
infer_entity_value_type_from_feature_views([Entity(name="id")], [fv1, fv2])
update_entities_with_inferred_types_from_feature_views(
[Entity(name="id")], [fv1, fv2]
)


@pytest.mark.integration
def test_infer_event_timestamp_column_for_data_source(simple_dataset_1):
def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1):
df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True)
df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"]

Expand Down