Skip to content

Schema Inferencing should happen at apply time #1646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 8 commits into from
Jun 18, 2021
Merged

Schema Inferencing should happen at apply time #1646

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ed8d461
wip1
mavysavydav Jun 12, 2021
91ef76b
just need to do clean up
mavysavydav Jun 12, 2021
0dab89c
linted
mavysavydav Jun 12, 2021
b24e571
improve test coverage
mavysavydav Jun 13, 2021
bb3a08a
changed placement of inference methods in repo_operation apply_total
mavysavydav Jun 15, 2021
3053599
updated inference method name + changed to void return since it updat…
mavysavydav Jun 15, 2021
d558b0b
fixed integration test and added comments
mavysavydav Jun 15, 2021
9a854bf
Made DataSource event_timestamp_column optional
mavysavydav Jun 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
updated inference method name + changed to void return since it updat… 
…es in place

Signed-off-by: David Y Liu <davidyliuliu@gmail.com>
  • Loading branch information
@mavysavydav
mavysavydav committed Jun 17, 2021
commit 30535997911f52cff26cf24e5425913b5bc5a4a9
4 changes: 2 additions & 2 deletions sdk/python/feast/feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
infer_event_timestamp_column_for_data_sources,
update_data_sources_with_inferred_event_timestamp_col,
)
from feast.infra.provider import Provider, RetrievalJob, get_provider
from feast.online_response import OnlineResponse, _infer_online_entity_rows
Expand Down Expand Up @@ -227,7 +227,7 @@ def apply(
entities_to_update = infer_entity_value_type_from_feature_views(
[ob for ob in objects if isinstance(ob, Entity)], views_to_update
)
infer_event_timestamp_column_for_data_sources(
update_data_sources_with_inferred_event_timestamp_col(
[view.input for view in views_to_update]
)
for view in views_to_update:
Expand Down
12 changes: 5 additions & 7 deletions sdk/python/feast/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import re
from typing import List
from typing import List, Union

from feast import Entity
from feast.data_source import BigQuerySource, DataSource, FileSource
from feast.data_source import BigQuerySource, FileSource
from feast.errors import RegistryInferenceFailure
from feast.feature_view import FeatureView
from feast.value_type import ValueType
Expand Down Expand Up @@ -60,9 +60,9 @@ def infer_entity_value_type_from_feature_views(
return entities


def infer_event_timestamp_column_for_data_sources(
data_sources: List[DataSource],
) -> List[DataSource]:
def update_data_sources_with_inferred_event_timestamp_col(
data_sources: List[Union[BigQuerySource, FileSource]],
) -> None:
ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column"

for data_source in data_sources:
Expand Down Expand Up @@ -115,5 +115,3 @@ def infer_event_timestamp_column_for_data_sources(
{ERROR_MSG_PREFIX} due to an absence of columns that satisfy the criteria.
""",
)

return data_sources
6 changes: 3 additions & 3 deletions sdk/python/feast/repo_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
infer_event_timestamp_column_for_data_sources,
update_data_sources_with_inferred_event_timestamp_col,
)
from feast.infra.offline_stores.helpers import assert_offline_store_supports_data_source
from feast.infra.provider import get_provider
Expand Down Expand Up @@ -160,9 +160,9 @@ def apply_total(repo_config: RepoConfig, repo_path: Path):
repo_config.offline_store, data_source
)

infer_event_timestamp_column_for_data_sources(data_sources)
update_data_sources_with_inferred_event_timestamp_col(data_sources)
for view in repo.feature_views:
view.infer_features_from_input_source()
view.infer_features_from_input_source()

tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
Expand Down
6 changes: 3 additions & 3 deletions sdk/python/tests/test_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from feast.feature_view import FeatureView
from feast.inference import (
infer_entity_value_type_from_feature_views,
infer_event_timestamp_column_for_data_sources,
update_data_sources_with_inferred_event_timestamp_col,
)


Expand Down Expand Up @@ -44,7 +44,7 @@ def test_infer_event_timestamp_column_for_data_source(simple_dataset_1):
df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"]

with prep_file_source(df=simple_dataset_1) as file_source:
actual_processed_data_sources = infer_event_timestamp_column_for_data_sources(
actual_processed_data_sources = update_data_sources_with_inferred_event_timestamp_col(
[
file_source,
simple_bq_source_using_table_ref_arg(simple_dataset_1),
Expand All @@ -60,4 +60,4 @@ def test_infer_event_timestamp_column_for_data_source(simple_dataset_1):
with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source:
with pytest.raises(RegistryInferenceFailure):
# two viable event_timestamp_columns
infer_event_timestamp_column_for_data_sources([file_source])
update_data_sources_with_inferred_event_timestamp_col([file_source])