Skip to content

Commit 53c5aaa

Browse files
jklegarjklegar
committed
Better logging for materialize command (#1499)
* Better logging for materialize command Signed-off-by: Jacob Klegar <jacob@tecton.ai> * Address comments Signed-off-by: Jacob Klegar <jacob@tecton.ai> * Move tqdm to FeatureStore class Signed-off-by: Jacob Klegar <jacob@tecton.ai> * Rebase Signed-off-by: Jacob Klegar <jacob@tecton.ai>
1 parent bae7932 commit 53c5aaa

5 files changed

Lines changed: 73 additions & 16 deletions

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import pandas as pd
2222
from colorama import Fore, Style
23+
from tqdm import tqdm
2324

2425
from feast import utils
2526
from feast.entity import Entity
@@ -341,6 +342,12 @@ def materialize_incremental(
341342
feature_view = self._registry.get_feature_view(name, self.project)
342343
feature_views_to_materialize.append(feature_view)
343344

345+
_print_materialization_log(
346+
None,
347+
end_date,
348+
len(feature_views_to_materialize),
349+
self.config.online_store.type,
350+
)
344351
# TODO paging large loads
345352
for feature_view in feature_views_to_materialize:
346353
start_date = feature_view.most_recent_end_time
@@ -352,11 +359,23 @@ def materialize_incremental(
352359
)
353360
start_date = datetime.utcnow() - feature_view.ttl
354361
provider = self._get_provider()
355-
_print_materialization_log(start_date, end_date, feature_view)
362+
print(
363+
f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}"
364+
f" from {Style.BRIGHT + Fore.GREEN}{start_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
365+
f" to {Style.BRIGHT + Fore.GREEN}{end_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}:"
366+
)
367+
368+
def tqdm_builder(length):
369+
return tqdm(total=length, ncols=100)
370+
356371
provider.materialize_single_feature_view(
357-
feature_view, start_date, end_date, self._registry, self.project
372+
feature_view,
373+
start_date,
374+
end_date,
375+
self._registry,
376+
self.project,
377+
tqdm_builder,
358378
)
359-
print(" done!")
360379

361380
def materialize(
362381
self,
@@ -406,14 +425,28 @@ def materialize(
406425
feature_view = self._registry.get_feature_view(name, self.project)
407426
feature_views_to_materialize.append(feature_view)
408427

428+
_print_materialization_log(
429+
start_date,
430+
end_date,
431+
len(feature_views_to_materialize),
432+
self.config.online_store.type,
433+
)
409434
# TODO paging large loads
410435
for feature_view in feature_views_to_materialize:
411436
provider = self._get_provider()
412-
_print_materialization_log(start_date, end_date, feature_view)
437+
print(f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}:")
438+
439+
def tqdm_builder(length):
440+
return tqdm(total=length, ncols=100)
441+
413442
provider.materialize_single_feature_view(
414-
feature_view, start_date, end_date, self._registry, self.project
443+
feature_view,
444+
start_date,
445+
end_date,
446+
self._registry,
447+
self.project,
448+
tqdm_builder,
415449
)
416-
print(" done!")
417450

418451
def get_online_features(
419452
self, feature_refs: List[str], entity_rows: List[Dict[str, Any]],
@@ -600,11 +633,19 @@ def _get_table_entity_keys(
600633
return entity_key_protos
601634

602635

603-
def _print_materialization_log(start_date, end_date, feature_view):
604-
print(
605-
f"Materializing feature view {Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}"
606-
f" from {Style.BRIGHT + Fore.GREEN}{start_date.astimezone()}{Style.RESET_ALL}"
607-
f" to {Style.BRIGHT + Fore.GREEN}{end_date.astimezone()}{Style.RESET_ALL}",
608-
end="",
609-
flush=True,
610-
)
636+
def _print_materialization_log(
637+
start_date, end_date, num_feature_views: int, online_store: str
638+
):
639+
if start_date:
640+
print(
641+
f"Materializing {Style.BRIGHT + Fore.GREEN}{num_feature_views}{Style.RESET_ALL} feature views"
642+
f" from {Style.BRIGHT + Fore.GREEN}{start_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
643+
f" to {Style.BRIGHT + Fore.GREEN}{end_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
644+
f" into the {Style.BRIGHT + Fore.GREEN}{online_store}{Style.RESET_ALL} online store.\n"
645+
)
646+
else:
647+
print(
648+
f"Materializing {Style.BRIGHT + Fore.GREEN}{num_feature_views}{Style.RESET_ALL} feature views"
649+
f" to {Style.BRIGHT + Fore.GREEN}{end_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
650+
f" into the {Style.BRIGHT + Fore.GREEN}{online_store}{Style.RESET_ALL} online store.\n"
651+
)

‎sdk/python/feast/infra/gcp.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import pandas
88
import pyarrow
99
from google.auth.exceptions import DefaultCredentialsError
10+
from tqdm import tqdm
1011

1112
from feast import FeatureTable, utils
1213
from feast.entity import Entity
@@ -146,6 +147,7 @@ def materialize_single_feature_view(
146147
end_date: datetime,
147148
registry: Registry,
148149
project: str,
150+
tqdm_builder: Callable[[int], tqdm],
149151
) -> None:
150152
entities = []
151153
for entity_name in feature_view.entities:
@@ -178,7 +180,10 @@ def materialize_single_feature_view(
178180
join_keys = [entity.join_key for entity in entities]
179181
rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)
180182

181-
self.online_write_batch(project, feature_view, rows_to_write, None)
183+
with tqdm_builder(len(rows_to_write)) as pbar:
184+
self.online_write_batch(
185+
project, feature_view, rows_to_write, lambda x: pbar.update(x)
186+
)
182187

183188
feature_view.materialization_intervals.append((start_date, end_date))
184189
registry.apply_feature_view(feature_view, project)

‎sdk/python/feast/infra/local.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import pandas as pd
88
import pytz
9+
from tqdm import tqdm
910

1011
from feast import FeatureTable, utils
1112
from feast.entity import Entity
@@ -123,6 +124,8 @@ def online_write_batch(
123124
created_ts,
124125
),
125126
)
127+
if progress:
128+
progress(1)
126129

127130
def online_read(
128131
self,
@@ -165,6 +168,7 @@ def materialize_single_feature_view(
165168
end_date: datetime,
166169
registry: Registry,
167170
project: str,
171+
tqdm_builder: Callable[[int], tqdm],
168172
) -> None:
169173
entities = []
170174
for entity_name in feature_view.entities:
@@ -197,7 +201,10 @@ def materialize_single_feature_view(
197201
join_keys = [entity.join_key for entity in entities]
198202
rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)
199203

200-
self.online_write_batch(project, feature_view, rows_to_write, None)
204+
with tqdm_builder(len(rows_to_write)) as pbar:
205+
self.online_write_batch(
206+
project, feature_view, rows_to_write, lambda x: pbar.update(x)
207+
)
201208

202209
feature_view.materialization_intervals.append((start_date, end_date))
203210
registry.apply_feature_view(feature_view, project)

‎sdk/python/feast/infra/provider.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import pandas
88
import pyarrow
9+
from tqdm import tqdm
910

1011
from feast import errors
1112
from feast.entity import Entity
@@ -102,6 +103,7 @@ def materialize_single_feature_view(
102103
end_date: datetime,
103104
registry: Registry,
104105
project: str,
106+
tqdm_builder: Callable[[int], tqdm],
105107
) -> None:
106108
pass
107109

‎sdk/python/tests/foo_provider.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
33

44
import pandas
5+
from tqdm import tqdm
56

67
from feast import Entity, FeatureTable, FeatureView, RepoConfig
78
from feast.infra.offline_stores.offline_store import RetrievalJob
@@ -49,6 +50,7 @@ def materialize_single_feature_view(
4950
end_date: datetime,
5051
registry: Registry,
5152
project: str,
53+
tqdm_builder: Callable[[int], tqdm],
5254
) -> None:
5355
pass
5456

0 commit comments

Comments
 (0)