Skip to content

Feast plan clean up #2256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 8 commits into from
Jan 30, 2022
Merged

Feast plan clean up #2256

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
817a8da
Run validation and inference on views and entities during plan
felixwang9817 Jan 29, 2022
22e3eec
Do not log objects that are unchanged
felixwang9817 Jan 29, 2022
6afdeba
Rename Fco to FeastObject
felixwang9817 Jan 29, 2022
8f5d9ef
Remove useless method
felixwang9817 Jan 29, 2022
4d5eb6b
Lint
felixwang9817 Jan 29, 2022
bc720f5
Always initialize registry during feature store initialization
felixwang9817 Jan 29, 2022
4e93615
Fix usage test
felixwang9817 Jan 29, 2022
335e8db
Remove print statements
felixwang9817 Jan 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove useless method 
Signed-off-by: Felix Wang <wangfelix98@gmail.com>
  • Loading branch information
@felixwang9817
felixwang9817 committed Jan 29, 2022
commit 8f5d9ef6b385c2e283de70e4f44fc05b50a9a235
33 changes: 15 additions & 18 deletions sdk/python/feast/diff/registry_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,6 @@ def tag_objects_for_keep_delete_update_add(
)


def tag_proto_objects_for_keep_delete_add(
existing_objs: Iterable[FeastObjectProto], desired_objs: Iterable[FeastObjectProto]
) -> Tuple[Iterable[FeastObjectProto], Iterable[FeastObjectProto], Iterable[FeastObjectProto]]:
existing_obj_names = {e.spec.name for e in existing_objs}
desired_obj_names = {e.spec.name for e in desired_objs}

objs_to_add = [e for e in desired_objs if e.spec.name not in existing_obj_names]
objs_to_keep = [e for e in desired_objs if e.spec.name in existing_obj_names]
objs_to_delete = [e for e in existing_objs if e.spec.name not in desired_obj_names]

return objs_to_keep, objs_to_delete, objs_to_add


FIELDS_TO_IGNORE = {"project"}


Expand Down Expand Up @@ -246,7 +233,9 @@ def diff_between(
)
for e in objects_to_update:
current_obj = [_e for _e in objects_to_keep if _e.name == e.name][0]
diff.add_feast_object_diff(diff_registry_objects(current_obj, e, object_type))
diff.add_feast_object_diff(
diff_registry_objects(current_obj, e, object_type)
)

return diff

Expand All @@ -268,7 +257,9 @@ def apply_diff_to_registry(
# will automatically delete the existing object.
if feast_object_diff.transition_type == TransitionType.DELETE:
if feast_object_diff.feast_object_type == FeastObjectType.ENTITY:
registry.delete_entity(feast_object_diff.current_feast_object.name, project, commit=False)
registry.delete_entity(
feast_object_diff.current_feast_object.name, project, commit=False
)
elif feast_object_diff.feast_object_type == FeastObjectType.FEATURE_SERVICE:
registry.delete_feature_service(
feast_object_diff.current_feast_object.name, project, commit=False
Expand All @@ -287,15 +278,21 @@ def apply_diff_to_registry(
TransitionType.UPDATE,
]:
if feast_object_diff.feast_object_type == FeastObjectType.ENTITY:
registry.apply_entity(feast_object_diff.new_feast_object, project, commit=False)
registry.apply_entity(
feast_object_diff.new_feast_object, project, commit=False
)
elif feast_object_diff.feast_object_type == FeastObjectType.FEATURE_SERVICE:
registry.apply_feature_service(feast_object_diff.new_feast_object, project, commit=False)
registry.apply_feature_service(
feast_object_diff.new_feast_object, project, commit=False
)
elif feast_object_diff.feast_object_type in [
FeastObjectType.FEATURE_VIEW,
FeastObjectType.ON_DEMAND_FEATURE_VIEW,
FeastObjectType.REQUEST_FEATURE_VIEW,
]:
registry.apply_feature_view(feast_object_diff.new_feast_object, project, commit=False)
registry.apply_feature_view(
feast_object_diff.new_feast_object, project, commit=False
)

if commit:
registry.commit()
59 changes: 12 additions & 47 deletions sdk/python/tests/unit/diff/test_registry_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,54 +1,11 @@
from feast.diff.registry_diff import (
diff_registry_objects,
tag_objects_for_keep_delete_update_add,
tag_proto_objects_for_keep_delete_add,
)
from feast.feature_view import FeatureView
from tests.utils.data_source_utils import prep_file_source


def test_tag_proto_objects_for_keep_delete_add(simple_dataset_1):
with prep_file_source(
df=simple_dataset_1, event_timestamp_column="ts_1"
) as file_source:
to_delete = FeatureView(
name="to_delete", entities=["id"], batch_source=file_source, ttl=None,
).to_proto()
unchanged_fv = FeatureView(
name="fv1", entities=["id"], batch_source=file_source, ttl=None,
).to_proto()
pre_changed = FeatureView(
name="fv2",
entities=["id"],
batch_source=file_source,
ttl=None,
tags={"when": "before"},
).to_proto()
post_changed = FeatureView(
name="fv2",
entities=["id"],
batch_source=file_source,
ttl=None,
tags={"when": "after"},
).to_proto()
to_add = FeatureView(
name="to_add", entities=["id"], batch_source=file_source, ttl=None,
).to_proto()

keep, delete, add = tag_proto_objects_for_keep_delete_add(
[unchanged_fv, pre_changed, to_delete], [unchanged_fv, post_changed, to_add]
)

assert len(list(keep)) == 2
assert unchanged_fv in keep
assert post_changed in keep
assert pre_changed not in keep
assert len(list(delete)) == 1
assert to_delete in delete
assert len(list(add)) == 1
assert to_add in add


def test_tag_objects_for_keep_delete_update_add(simple_dataset_1):
with prep_file_source(
df=simple_dataset_1, event_timestamp_column="ts_1"
Expand Down Expand Up @@ -114,12 +71,20 @@ def test_diff_registry_objects_feature_views(simple_dataset_1):
tags={"when": "after"},
)

feast_object_diffs = diff_registry_objects(pre_changed, pre_changed, "feature view")
feast_object_diffs = diff_registry_objects(
pre_changed, pre_changed, "feature view"
)
assert len(feast_object_diffs.feast_object_property_diffs) == 0

feast_object_diffs = diff_registry_objects(pre_changed, post_changed, "feature view")
feast_object_diffs = diff_registry_objects(
pre_changed, post_changed, "feature view"
)
assert len(feast_object_diffs.feast_object_property_diffs) == 1

assert feast_object_diffs.feast_object_property_diffs[0].property_name == "tags"
assert feast_object_diffs.feast_object_property_diffs[0].val_existing == {"when": "before"}
assert feast_object_diffs.feast_object_property_diffs[0].val_declared == {"when": "after"}
assert feast_object_diffs.feast_object_property_diffs[0].val_existing == {
"when": "before"
}
assert feast_object_diffs.feast_object_property_diffs[0].val_declared == {
"when": "after"
}