From 8efc9cd41ef2b845d45d12d848a9d1520a53ba0d Mon Sep 17 00:00:00 2001 From: David Y Liu Date: Fri, 25 Jun 2021 10:32:16 -0700 Subject: [PATCH 1/2] grouped inferencing statements together Signed-off-by: David Y Liu --- sdk/python/feast/feature_store.py | 9 ++++++--- sdk/python/feast/inference.py | 6 ++---- sdk/python/feast/repo_operations.py | 30 +++++++++++++---------------- sdk/python/tests/test_inference.py | 17 ++++++++-------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e34d314e5fd..d57808ac735 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -27,8 +27,8 @@ from feast.errors import FeastProviderLoginError, FeatureViewNotFoundException from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.online_response import OnlineResponse, _infer_online_entity_rows @@ -224,8 +224,11 @@ def apply( assert isinstance(objects, list) views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)] - entities_to_update = infer_entity_value_type_from_feature_views( - [ob for ob in objects if isinstance(ob, Entity)], views_to_update + entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] + + # Make inferences + update_entities_with_inferred_types_from_feature_views( + entities_to_update, views_to_update ) update_data_sources_with_inferred_event_timestamp_col( [view.input for view in views_to_update] diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index fac2155ee21..af95f9d2551 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -8,9 +8,9 @@ from feast.value_type import ValueType -def infer_entity_value_type_from_feature_views( +def update_entities_with_inferred_types_from_feature_views( entities: List[Entity], feature_views: List[FeatureView] -) -> List[Entity]: +) -> None: """ Infer entity value type by examining schema of feature view input sources """ @@ -57,8 +57,6 @@ def infer_entity_value_type_from_feature_views( entity.value_type = inferred_value_type - return entities - def update_data_sources_with_inferred_event_timestamp_col( data_sources: List[Union[BigQuerySource, FileSource]], diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 4f2cb1981d3..f707f19c41b 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -14,8 +14,8 @@ from feast import Entity, FeatureTable from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) from feast.infra.provider import get_provider from feast.names import adjectives, animals @@ -131,13 +131,19 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) - repo = ParsedRepo( - feature_tables=repo.feature_tables, - entities=infer_entity_value_type_from_feature_views( - repo.entities, repo.feature_views - ), - feature_views=repo.feature_views, + data_sources = [t.input for t in repo.feature_views] + + # Make sure the data source used by this feature view is supported by Feast + for data_source in data_sources: + data_source.validate() + + # Make inferences + update_entities_with_inferred_types_from_feature_views( + repo.entities, repo.feature_views ) + update_data_sources_with_inferred_event_timestamp_col(data_sources) + for view in repo.feature_views: + view.infer_features_from_input_source() sys.dont_write_bytecode = False for entity in repo.entities: @@ -151,16 +157,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): for t in repo.feature_views: repo_table_names.add(t.name) - data_sources = [t.input for t in repo.feature_views] - - # Make sure the data source used by this feature view is supported by Feast - for data_source in data_sources: - data_source.validate() - - update_data_sources_with_inferred_event_timestamp_col(data_sources) - for view in repo.feature_views: - view.infer_features_from_input_source() - tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: diff --git a/sdk/python/tests/test_inference.py b/sdk/python/tests/test_inference.py index 1f626ac3cd3..507283b2afd 100644 --- a/sdk/python/tests/test_inference.py +++ b/sdk/python/tests/test_inference.py @@ -9,8 +9,8 @@ from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) @@ -24,18 +24,19 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat fv1 = FeatureView(name="fv1", entities=["id"], input=file_source, ttl=None,) fv2 = FeatureView(name="fv2", entities=["id"], input=file_source_2, ttl=None,) - actual_1 = infer_entity_value_type_from_feature_views( - [Entity(name="id")], [fv1] - ) - actual_2 = infer_entity_value_type_from_feature_views( - [Entity(name="id")], [fv2] - ) + actual_1 = Entity(name="id") + actual_2 = Entity(name="id") + + update_entities_with_inferred_types_from_feature_views([actual_1], [fv1]) + update_entities_with_inferred_types_from_feature_views([actual_2], [fv2]) assert actual_1 == [Entity(name="id", value_type=ValueType.INT64)] assert actual_2 == [Entity(name="id", value_type=ValueType.STRING)] with pytest.raises(RegistryInferenceFailure): # two viable data types - infer_entity_value_type_from_feature_views([Entity(name="id")], [fv1, fv2]) + update_entities_with_inferred_types_from_feature_views( + [Entity(name="id")], [fv1, fv2] + ) @pytest.mark.integration From 9b2c8db70d5089c2c1b7be43cc92ea1b0c6f1fab Mon Sep 17 00:00:00 2001 From: David Y Liu Date: Fri, 25 Jun 2021 10:43:45 -0700 Subject: [PATCH 2/2] update in testing Signed-off-by: David Y Liu --- sdk/python/tests/test_inference.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/python/tests/test_inference.py b/sdk/python/tests/test_inference.py index 507283b2afd..9405bce1f28 100644 --- a/sdk/python/tests/test_inference.py +++ b/sdk/python/tests/test_inference.py @@ -14,7 +14,9 @@ ) -def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dataset_2): +def test_update_entities_with_inferred_types_from_feature_views( + simple_dataset_1, simple_dataset_2 +): with prep_file_source( df=simple_dataset_1, event_timestamp_column="ts_1" ) as file_source, prep_file_source( @@ -29,8 +31,8 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat update_entities_with_inferred_types_from_feature_views([actual_1], [fv1]) update_entities_with_inferred_types_from_feature_views([actual_2], [fv2]) - assert actual_1 == [Entity(name="id", value_type=ValueType.INT64)] - assert actual_2 == [Entity(name="id", value_type=ValueType.STRING)] + assert actual_1 == Entity(name="id", value_type=ValueType.INT64) + assert actual_2 == Entity(name="id", value_type=ValueType.STRING) with pytest.raises(RegistryInferenceFailure): # two viable data types @@ -40,7 +42,7 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat @pytest.mark.integration -def test_infer_event_timestamp_column_for_data_source(simple_dataset_1): +def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1): df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True) df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"]