Skip to content

Commit 54d0f3a

Browse files
felixwang9817felixwang9817
authored
Duplicate feast apply bug (#2087)
* Refactor feast registry-dump cli command Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Test that running feast apply twice does not change registry contents Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Ensure that Registry.to_dict sorts contents Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Do not remove dummy entity Signed-off-by: Felix Wang <wangfelix98@gmail.com>
1 parent 0356181 commit 54d0f3a

3 files changed

Lines changed: 70 additions & 29 deletions

File tree

‎sdk/python/feast/registry.py‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
from collections import defaultdict
1516
from datetime import datetime, timedelta
1617
from pathlib import Path
17-
from typing import Dict, List, Optional
18+
from typing import Any, Dict, List, Optional
1819
from urllib.parse import urlparse
1920

2021
from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
22+
from google.protobuf.json_format import MessageToDict
2123
from proto import Message
2224

2325
from feast import importer
@@ -671,6 +673,56 @@ def teardown(self):
671673
"""Tears down (removes) the registry."""
672674
self._registry_store.teardown()
673675

676+
def to_dict(self, project: str) -> Dict[str, List[Any]]:
677+
"""Returns a dictionary representation of the registry contents for the specified project.
678+
679+
For each list in the dictionary, the elements are sorted by name, so this
680+
method can be used to compare two registries.
681+
682+
Args:
683+
project: Feast project to convert to a dict
684+
"""
685+
registry_dict = defaultdict(list)
686+
687+
for entity in sorted(
688+
self.list_entities(project=project), key=lambda entity: entity.name
689+
):
690+
registry_dict["entities"].append(MessageToDict(entity.to_proto()))
691+
for feature_view in sorted(
692+
self.list_feature_views(project=project),
693+
key=lambda feature_view: feature_view.name,
694+
):
695+
registry_dict["featureViews"].append(MessageToDict(feature_view.to_proto()))
696+
for feature_table in sorted(
697+
self.list_feature_tables(project=project),
698+
key=lambda feature_table: feature_table.name,
699+
):
700+
registry_dict["featureTables"].append(
701+
MessageToDict(feature_table.to_proto())
702+
)
703+
for feature_service in sorted(
704+
self.list_feature_services(project=project),
705+
key=lambda feature_service: feature_service.name,
706+
):
707+
registry_dict["featureServices"].append(
708+
MessageToDict(feature_service.to_proto())
709+
)
710+
for on_demand_feature_view in sorted(
711+
self.list_on_demand_feature_views(project=project),
712+
key=lambda on_demand_feature_view: on_demand_feature_view.name,
713+
):
714+
registry_dict["onDemandFeatureViews"].append(
715+
MessageToDict(on_demand_feature_view.to_proto())
716+
)
717+
for request_feature_view in sorted(
718+
self.list_request_feature_views(project=project),
719+
key=lambda request_feature_view: request_feature_view.name,
720+
):
721+
registry_dict["requestFeatureViews"].append(
722+
MessageToDict(request_feature_view.to_proto())
723+
)
724+
return registry_dict
725+
674726
def _prepare_registry_for_changes(self):
675727
"""Prepares the Registry for changes by refreshing the cache if necessary."""
676728
try:

‎sdk/python/feast/repo_operations.py‎

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,18 @@
44
import random
55
import re
66
import sys
7-
from collections import defaultdict
87
from importlib.abc import Loader
98
from pathlib import Path
109
from typing import List, NamedTuple, Set, Tuple, Union, cast
1110

1211
import click
1312
from click.exceptions import BadParameter
14-
from google.protobuf.json_format import MessageToDict
1513

1614
from feast import Entity, FeatureTable
1715
from feast.base_feature_view import BaseFeatureView
1816
from feast.feature_service import FeatureService
1917
from feast.feature_store import FeatureStore
20-
from feast.feature_view import FeatureView
18+
from feast.feature_view import DUMMY_ENTITY_NAME, FeatureView
2119
from feast.names import adjectives, animals
2220
from feast.on_demand_feature_view import OnDemandFeatureView
2321
from feast.registry import Registry
@@ -267,7 +265,11 @@ def _tag_registry_entities_for_keep_delete(
267265
entities_to_delete: Set[Entity] = set()
268266
repo_entities_names = set([e.name for e in repo.entities])
269267
for registry_entity in registry.list_entities(project=project):
270-
if registry_entity.name not in repo_entities_names:
268+
# Do not delete dummy entity.
269+
if (
270+
registry_entity.name not in repo_entities_names
271+
and registry_entity.name != DUMMY_ENTITY_NAME
272+
):
271273
entities_to_delete.add(registry_entity)
272274
return entities_to_keep, entities_to_delete
273275

@@ -339,28 +341,8 @@ def registry_dump(repo_config: RepoConfig, repo_path: Path):
339341
registry_config = repo_config.get_registry_config()
340342
project = repo_config.project
341343
registry = Registry(registry_config=registry_config, repo_path=repo_path)
342-
registry_dict = defaultdict(list)
343-
344-
for entity in registry.list_entities(project=project):
345-
registry_dict["entities"].append(MessageToDict(entity.to_proto()))
346-
for feature_view in registry.list_feature_views(project=project):
347-
registry_dict["featureViews"].append(MessageToDict(feature_view.to_proto()))
348-
for feature_table in registry.list_feature_tables(project=project):
349-
registry_dict["featureTables"].append(MessageToDict(feature_table.to_proto()))
350-
for feature_service in registry.list_feature_services(project=project):
351-
registry_dict["featureServices"].append(
352-
MessageToDict(feature_service.to_proto())
353-
)
354-
for on_demand_feature_view in registry.list_on_demand_feature_views(
355-
project=project
356-
):
357-
registry_dict["onDemandFeatureViews"].append(
358-
MessageToDict(on_demand_feature_view.to_proto())
359-
)
360-
for request_feature_view in registry.list_request_feature_views(project=project):
361-
registry_dict["requestFeatureViews"].append(
362-
MessageToDict(request_feature_view.to_proto())
363-
)
344+
registry_dict = registry.to_dict(project=project)
345+
364346
warning = (
365347
"Warning: The registry-dump command is for debugging only and may contain "
366348
"breaking changes in the future. No guarantees are made on this interface."

‎sdk/python/tests/integration/registration/test_cli.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ def test_universal_cli(test_repo_config) -> None:
3939
result = runner.run(["apply"], cwd=repo_path)
4040
assertpy.assert_that(result.returncode).is_equal_to(0)
4141

42+
# Store registry contents, to be compared later.
43+
fs = FeatureStore(repo_path=str(repo_path))
44+
registry_dict = fs.registry.to_dict(project=project)
45+
4246
# entity & feature view list commands should succeed
4347
result = runner.run(["entities", "list"], cwd=repo_path)
4448
assertpy.assert_that(result.returncode).is_equal_to(0)
@@ -58,8 +62,6 @@ def test_universal_cli(test_repo_config) -> None:
5862
["feature-services", "describe", "driver_locations_service"], cwd=repo_path
5963
)
6064
assertpy.assert_that(result.returncode).is_equal_to(0)
61-
62-
fs = FeatureStore(repo_path=str(repo_path))
6365
assertpy.assert_that(fs.list_feature_views()).is_length(3)
6466

6567
# entity & feature view describe commands should fail when objects don't exist
@@ -78,6 +80,11 @@ def test_universal_cli(test_repo_config) -> None:
7880
view_name="driver_locations",
7981
)
8082

83+
# Confirm that registry contents have not changed.
84+
assertpy.assert_that(registry_dict).is_equal_to(
85+
fs.registry.to_dict(project=project)
86+
)
87+
8188
result = runner.run(["teardown"], cwd=repo_path)
8289
assertpy.assert_that(result.returncode).is_equal_to(0)
8390

0 commit comments

Comments
 (0)