Skip to content

Commit 3dcec6d

Browse files
felixwang9817felixwang9817
authored
Compare Python objects instead of proto objects (feast-dev#2227)
* Compare Python objects instead of proto objects Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Remove unnecessary helper method Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix docstring test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add docstring to RepoContents Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Lint Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Update usage test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Set cache ttl to 1 second in tests for local feature server tests Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add FCO test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add properties to feature service Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Lint Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Remove logic that converts Registry to RepoContents Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Always initialize registry Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Move diffing methods from Registry into FcoDiff.py Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix unit test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Put registry initialization back in repo_operations.py Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix usage test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Switch from hardcoded names to enum Signed-off-by: Felix Wang <wangfelix98@gmail.com>
1 parent 3894b8d commit 3dcec6d

File tree

8 files changed

+408
-264
lines changed

8 files changed

+408
-264
lines changed

‎sdk/python/feast/diff/FcoDiff.py‎

Lines changed: 175 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from dataclasses import dataclass
2-
from typing import Generic, Iterable, List, Set, Tuple, TypeVar
2+
from typing import Any, Dict, Generic, Iterable, List, Set, Tuple, TypeVar
33

44
from feast.base_feature_view import BaseFeatureView
55
from feast.diff.property_diff import PropertyDiff, TransitionType
@@ -16,23 +16,28 @@
1616
from feast.protos.feast.core.RequestFeatureView_pb2 import (
1717
RequestFeatureView as RequestFeatureViewProto,
1818
)
19+
from feast.registry import FeastObjectType, Registry
20+
from feast.repo_contents import RepoContents
1921

20-
FcoProto = TypeVar(
21-
"FcoProto",
22-
EntityProto,
23-
FeatureViewProto,
24-
FeatureServiceProto,
25-
OnDemandFeatureViewProto,
26-
RequestFeatureViewProto,
27-
)
22+
FEAST_OBJECT_TYPE_TO_STR = {
23+
FeastObjectType.ENTITY: "entity",
24+
FeastObjectType.FEATURE_VIEW: "feature view",
25+
FeastObjectType.ON_DEMAND_FEATURE_VIEW: "on demand feature view",
26+
FeastObjectType.REQUEST_FEATURE_VIEW: "request feature view",
27+
FeastObjectType.FEATURE_SERVICE: "feature service",
28+
}
29+
30+
FEAST_OBJECT_TYPES = FEAST_OBJECT_TYPE_TO_STR.keys()
31+
32+
Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)
2833

2934

3035
@dataclass
31-
class FcoDiff(Generic[FcoProto]):
36+
class FcoDiff(Generic[Fco]):
3237
name: str
3338
fco_type: str
34-
current_fco: FcoProto
35-
new_fco: FcoProto
39+
current_fco: Fco
40+
new_fco: Fco
3641
fco_property_diffs: List[PropertyDiff]
3742
transition_type: TransitionType
3843

@@ -48,20 +53,28 @@ def add_fco_diff(self, fco_diff: FcoDiff):
4853
self.fco_diffs.append(fco_diff)
4954

5055

51-
Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)
52-
53-
54-
def tag_objects_for_keep_delete_add(
56+
def tag_objects_for_keep_delete_update_add(
5557
existing_objs: Iterable[Fco], desired_objs: Iterable[Fco]
56-
) -> Tuple[Set[Fco], Set[Fco], Set[Fco]]:
58+
) -> Tuple[Set[Fco], Set[Fco], Set[Fco], Set[Fco]]:
5759
existing_obj_names = {e.name for e in existing_objs}
5860
desired_obj_names = {e.name for e in desired_objs}
5961

6062
objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names}
61-
objs_to_keep = {e for e in desired_objs if e.name in existing_obj_names}
63+
objs_to_update = {e for e in desired_objs if e.name in existing_obj_names}
64+
objs_to_keep = {e for e in existing_objs if e.name in desired_obj_names}
6265
objs_to_delete = {e for e in existing_objs if e.name not in desired_obj_names}
6366

64-
return objs_to_keep, objs_to_delete, objs_to_add
67+
return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add
68+
69+
70+
FcoProto = TypeVar(
71+
"FcoProto",
72+
EntityProto,
73+
FeatureViewProto,
74+
FeatureServiceProto,
75+
OnDemandFeatureViewProto,
76+
RequestFeatureViewProto,
77+
)
6578

6679

6780
def tag_proto_objects_for_keep_delete_add(
@@ -80,23 +93,158 @@ def tag_proto_objects_for_keep_delete_add(
8093
FIELDS_TO_IGNORE = {"project"}
8194

8295

83-
def diff_between(current: FcoProto, new: FcoProto, object_type: str) -> FcoDiff:
84-
assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name
96+
def diff_registry_objects(current: Fco, new: Fco, object_type: str) -> FcoDiff:
97+
current_proto = current.to_proto()
98+
new_proto = new.to_proto()
99+
assert current_proto.DESCRIPTOR.full_name == new_proto.DESCRIPTOR.full_name
85100
property_diffs = []
86101
transition: TransitionType = TransitionType.UNCHANGED
87-
if current.spec != new.spec:
88-
for _field in current.spec.DESCRIPTOR.fields:
102+
if current_proto.spec != new_proto.spec:
103+
for _field in current_proto.spec.DESCRIPTOR.fields:
89104
if _field.name in FIELDS_TO_IGNORE:
90105
continue
91-
if getattr(current.spec, _field.name) != getattr(new.spec, _field.name):
106+
if getattr(current_proto.spec, _field.name) != getattr(
107+
new_proto.spec, _field.name
108+
):
92109
transition = TransitionType.UPDATE
93110
property_diffs.append(
94111
PropertyDiff(
95112
_field.name,
96-
getattr(current.spec, _field.name),
97-
getattr(new.spec, _field.name),
113+
getattr(current_proto.spec, _field.name),
114+
getattr(new_proto.spec, _field.name),
98115
)
99116
)
100117
return FcoDiff(
101-
new.spec.name, object_type, current, new, property_diffs, transition,
118+
name=new_proto.spec.name,
119+
fco_type=object_type,
120+
current_fco=current,
121+
new_fco=new,
122+
fco_property_diffs=property_diffs,
123+
transition_type=transition,
102124
)
125+
126+
127+
def extract_objects_for_keep_delete_update_add(
128+
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
129+
) -> Tuple[
130+
Dict[FeastObjectType, Set[Fco]],
131+
Dict[FeastObjectType, Set[Fco]],
132+
Dict[FeastObjectType, Set[Fco]],
133+
Dict[FeastObjectType, Set[Fco]],
134+
]:
135+
"""
136+
Returns the objects in the registry that must be modified to achieve the desired repo state.
137+
138+
Args:
139+
registry: The registry storing the current repo state.
140+
current_project: The Feast project whose objects should be compared.
141+
desired_repo_contents: The desired repo state.
142+
"""
143+
objs_to_keep = {}
144+
objs_to_delete = {}
145+
objs_to_update = {}
146+
objs_to_add = {}
147+
148+
registry_object_type_to_objects: Dict[FeastObjectType, List[Any]]
149+
registry_object_type_to_objects = {
150+
FeastObjectType.ENTITY: registry.list_entities(project=current_project),
151+
FeastObjectType.FEATURE_VIEW: registry.list_feature_views(
152+
project=current_project
153+
),
154+
FeastObjectType.ON_DEMAND_FEATURE_VIEW: registry.list_on_demand_feature_views(
155+
project=current_project
156+
),
157+
FeastObjectType.REQUEST_FEATURE_VIEW: registry.list_request_feature_views(
158+
project=current_project
159+
),
160+
FeastObjectType.FEATURE_SERVICE: registry.list_feature_services(
161+
project=current_project
162+
),
163+
}
164+
registry_object_type_to_repo_contents: Dict[FeastObjectType, Set[Any]]
165+
registry_object_type_to_repo_contents = {
166+
FeastObjectType.ENTITY: desired_repo_contents.entities,
167+
FeastObjectType.FEATURE_VIEW: desired_repo_contents.feature_views,
168+
FeastObjectType.ON_DEMAND_FEATURE_VIEW: desired_repo_contents.on_demand_feature_views,
169+
FeastObjectType.REQUEST_FEATURE_VIEW: desired_repo_contents.request_feature_views,
170+
FeastObjectType.FEATURE_SERVICE: desired_repo_contents.feature_services,
171+
}
172+
173+
for object_type in FEAST_OBJECT_TYPES:
174+
(
175+
to_keep,
176+
to_delete,
177+
to_update,
178+
to_add,
179+
) = tag_objects_for_keep_delete_update_add(
180+
registry_object_type_to_objects[object_type],
181+
registry_object_type_to_repo_contents[object_type],
182+
)
183+
184+
objs_to_keep[object_type] = to_keep
185+
objs_to_delete[object_type] = to_delete
186+
objs_to_update[object_type] = to_update
187+
objs_to_add[object_type] = to_add
188+
189+
return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add
190+
191+
192+
def diff_between(
193+
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
194+
) -> RegistryDiff:
195+
"""
196+
Returns the difference between the current and desired repo states.
197+
198+
Args:
199+
registry: The registry storing the current repo state.
200+
current_project: The Feast project for which the diff is being computed.
201+
desired_repo_contents: The desired repo state.
202+
"""
203+
diff = RegistryDiff()
204+
205+
(
206+
objs_to_keep,
207+
objs_to_delete,
208+
objs_to_update,
209+
objs_to_add,
210+
) = extract_objects_for_keep_delete_update_add(
211+
registry, current_project, desired_repo_contents
212+
)
213+
214+
for object_type in FEAST_OBJECT_TYPES:
215+
objects_to_keep = objs_to_keep[object_type]
216+
objects_to_delete = objs_to_delete[object_type]
217+
objects_to_update = objs_to_update[object_type]
218+
objects_to_add = objs_to_add[object_type]
219+
220+
for e in objects_to_add:
221+
diff.add_fco_diff(
222+
FcoDiff(
223+
name=e.name,
224+
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
225+
current_fco=None,
226+
new_fco=e,
227+
fco_property_diffs=[],
228+
transition_type=TransitionType.CREATE,
229+
)
230+
)
231+
for e in objects_to_delete:
232+
diff.add_fco_diff(
233+
FcoDiff(
234+
name=e.name,
235+
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
236+
current_fco=e,
237+
new_fco=None,
238+
fco_property_diffs=[],
239+
transition_type=TransitionType.DELETE,
240+
)
241+
)
242+
for e in objects_to_update:
243+
current_obj = [_e for _e in objects_to_keep if _e.name == e.name][0]
244+
diff.add_fco_diff(
245+
diff_registry_objects(
246+
current_obj, e, FEAST_OBJECT_TYPE_TO_STR[object_type]
247+
)
248+
)
249+
250+
return diff

‎sdk/python/feast/feature_service.py‎

Lines changed: 63 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ class FeatureService:
3030
Services.
3131
"""
3232

33-
name: str
34-
feature_view_projections: List[FeatureViewProjection]
35-
tags: Dict[str, str]
36-
description: Optional[str] = None
37-
created_timestamp: Optional[datetime] = None
38-
last_updated_timestamp: Optional[datetime] = None
33+
_name: str
34+
_feature_view_projections: List[FeatureViewProjection]
35+
_tags: Dict[str, str]
36+
_description: Optional[str] = None
37+
_created_timestamp: Optional[datetime] = None
38+
_last_updated_timestamp: Optional[datetime] = None
3939

4040
@log_exceptions
4141
def __init__(
@@ -51,22 +51,22 @@ def __init__(
5151
Raises:
5252
ValueError: If one of the specified features is not a valid type.
5353
"""
54-
self.name = name
55-
self.feature_view_projections = []
54+
self._name = name
55+
self._feature_view_projections = []
5656

5757
for feature_grouping in features:
5858
if isinstance(feature_grouping, BaseFeatureView):
59-
self.feature_view_projections.append(feature_grouping.projection)
59+
self._feature_view_projections.append(feature_grouping.projection)
6060
else:
6161
raise ValueError(
6262
"The FeatureService {fs_name} has been provided with an invalid type"
6363
f'{type(feature_grouping)} as part of the "features" argument.)'
6464
)
6565

66-
self.tags = tags or {}
67-
self.description = description
68-
self.created_timestamp = None
69-
self.last_updated_timestamp = None
66+
self._tags = tags or {}
67+
self._description = description
68+
self._created_timestamp = None
69+
self._last_updated_timestamp = None
7070

7171
def __repr__(self):
7272
items = (f"{k} = {v}" for k, v in self.__dict__.items())
@@ -93,6 +93,56 @@ def __eq__(self, other):
9393

9494
return True
9595

96+
@property
97+
def name(self) -> str:
98+
return self._name
99+
100+
@name.setter
101+
def name(self, name: str):
102+
self._name = name
103+
104+
@property
105+
def feature_view_projections(self) -> List[FeatureViewProjection]:
106+
return self._feature_view_projections
107+
108+
@feature_view_projections.setter
109+
def feature_view_projections(
110+
self, feature_view_projections: List[FeatureViewProjection]
111+
):
112+
self._feature_view_projections = feature_view_projections
113+
114+
@property
115+
def tags(self) -> Dict[str, str]:
116+
return self._tags
117+
118+
@tags.setter
119+
def tags(self, tags: Dict[str, str]):
120+
self._tags = tags
121+
122+
@property
123+
def description(self) -> Optional[str]:
124+
return self._description
125+
126+
@description.setter
127+
def description(self, description: str):
128+
self._description = description
129+
130+
@property
131+
def created_timestamp(self) -> Optional[datetime]:
132+
return self._created_timestamp
133+
134+
@created_timestamp.setter
135+
def created_timestamp(self, created_timestamp: datetime):
136+
self._created_timestamp = created_timestamp
137+
138+
@property
139+
def last_updated_timestamp(self) -> Optional[datetime]:
140+
return self._last_updated_timestamp
141+
142+
@last_updated_timestamp.setter
143+
def last_updated_timestamp(self, last_updated_timestamp: datetime):
144+
self._last_updated_timestamp = last_updated_timestamp
145+
96146
@staticmethod
97147
def from_proto(feature_service_proto: FeatureServiceProto):
98148
"""

0 commit comments

Comments
 (0)