Skip to content

Commit a3304d4

Browse files
achalsachals
authored
feat: Add feast repo-upgrade for automated repo upgrades (feast-dev#2733)
* WIP feat: Add feast repo-upgrade for automated repo upgrades Signed-off-by: Achal Shah <achals@gmail.com> * pin deps Signed-off-by: Achal Shah <achals@gmail.com>
1 parent bee8076 commit a3304d4

File tree

9 files changed

+269
-107
lines changed

9 files changed

+269
-107
lines changed

‎sdk/python/feast/cli.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
registry_dump,
4242
teardown,
4343
)
44+
from feast.repo_upgrade import RepoUpgrader
4445
from feast.utils import maybe_local_tz
4546

4647
_logger = logging.getLogger(__name__)
@@ -85,7 +86,7 @@ def cli(ctx: click.Context, chdir: Optional[str], log_level: str):
8586
try:
8687
level = getattr(logging, log_level.upper())
8788
logging.basicConfig(
88-
format="%(asctime)s %(levelname)s:%(message)s",
89+
format="%(asctime)s %(name)s %(levelname)s: %(message)s",
8990
datefmt="%m/%d/%Y %I:%M:%S %p",
9091
level=level,
9192
)
@@ -98,7 +99,6 @@ def cli(ctx: click.Context, chdir: Optional[str], log_level: str):
9899
if "feast" in logger_name:
99100
logger = logging.getLogger(logger_name)
100101
logger.setLevel(level)
101-
102102
except Exception as e:
103103
raise e
104104
pass
@@ -825,5 +825,25 @@ def validate(
825825
exit(1)
826826

827827

828+
@cli.command("repo-upgrade", cls=NoOptionDefaultFormat)
829+
@click.option(
830+
"--write",
831+
is_flag=True,
832+
default=False,
833+
help="Upgrade a feature repo to use the API expected by feast 0.23.",
834+
)
835+
@click.pass_context
836+
def repo_upgrade(ctx: click.Context, write: bool):
837+
"""
838+
Upgrade a feature repo in place.
839+
"""
840+
repo = ctx.obj["CHDIR"]
841+
cli_check_repo(repo)
842+
try:
843+
RepoUpgrader(repo, write).upgrade()
844+
except FeastProviderLoginError as e:
845+
print(str(e))
846+
847+
828848
if __name__ == "__main__":
829849
cli()

‎sdk/python/feast/repo_upgrade.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import logging
2+
from pathlib import Path
3+
from typing import Dict, List
4+
5+
from bowler import Query
6+
from fissix.pgen2 import token
7+
from fissix.pygram import python_symbols
8+
from fissix.pytree import Node
9+
10+
from feast.repo_operations import get_repo_files
11+
12+
SOURCES = {
13+
"FileSource",
14+
"BigQuerySource",
15+
"RedshiftSource",
16+
"SnowflakeSource",
17+
"KafkaSource",
18+
"KinesisSource",
19+
}
20+
21+
22+
class RepoUpgrader:
23+
def __init__(self, repo_path: str, write: bool):
24+
self.repo_path = repo_path
25+
self.write = write
26+
self.repo_files: List[str] = [
27+
str(p) for p in get_repo_files(Path(self.repo_path))
28+
]
29+
logging.getLogger("RefactoringTool").setLevel(logging.WARNING)
30+
31+
def upgrade(self):
32+
self.remove_date_partition_column()
33+
34+
def remove_date_partition_column(self):
35+
def _remove_date_partition_column(
36+
node: Node, capture: Dict[str, Node], filename: str
37+
) -> None:
38+
self.remove_argument_transform(node, "date_partition_column")
39+
40+
for s in SOURCES:
41+
Query(self.repo_files).select_class(s).is_call().modify(
42+
_remove_date_partition_column
43+
).execute(write=self.write, interactive=False)
44+
45+
@staticmethod
46+
def remove_argument_transform(node: Node, argument: str):
47+
"""
48+
Removes the specified argument.
49+
For example, if the argument is "join_key", this method transforms
50+
driver = Entity(
51+
name="driver_id",
52+
join_key="driver_id",
53+
)
54+
into
55+
driver = Entity(
56+
name="driver_id",
57+
)
58+
This method assumes that node represents a class call that already has an arglist.
59+
"""
60+
if len(node.children) < 2 or len(node.children[1].children) < 2:
61+
raise ValueError(f"Expected a class call with an arglist but got {node}.")
62+
class_args = node.children[1].children[1].children
63+
for i, class_arg in enumerate(class_args):
64+
if (
65+
class_arg.type == python_symbols.argument
66+
and class_arg.children[0].value == argument
67+
):
68+
class_args.pop(i)
69+
if i < len(class_args) and class_args[i].type == token.COMMA:
70+
class_args.pop(i)
71+
if i < len(class_args) and class_args[i].type == token.NEWLINE:
72+
class_args.pop(i)

‎sdk/python/requirements/py3.10-ci-requirements.txt‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,13 @@ altair==4.2.0
3131
anyio==3.6.1
3232
# via
3333
# starlette
34-
# watchgod
34+
# watchfiles
3535
appdirs==1.4.4
36-
# via black
36+
# via
37+
# black
38+
# fissix
3739
appnope==0.1.3
3840
# via ipython
39-
asgiref==3.5.2
40-
# via uvicorn
4141
asn1crypto==1.5.1
4242
# via
4343
# oscrypto
@@ -54,6 +54,7 @@ attrs==21.4.0
5454
# via
5555
# aiohttp
5656
# black
57+
# bowler
5758
# jsonschema
5859
# pytest
5960
avro==1.10.0
@@ -86,6 +87,8 @@ botocore==1.23.24
8687
# boto3
8788
# moto
8889
# s3transfer
90+
bowler==0.9.0
91+
# via feast (setup.py)
8992
build==0.8.0
9093
# via feast (setup.py)
9194
cachecontrol==0.12.11
@@ -113,8 +116,10 @@ charset-normalizer==2.0.12
113116
click==8.0.1
114117
# via
115118
# black
119+
# bowler
116120
# feast (setup.py)
117121
# great-expectations
122+
# moreorless
118123
# pip-tools
119124
# uvicorn
120125
cloudpickle==2.1.0
@@ -178,6 +183,8 @@ filelock==3.7.1
178183
# via virtualenv
179184
firebase-admin==4.5.2
180185
# via feast (setup.py)
186+
fissix==21.11.13
187+
# via bowler
181188
flake8==4.0.1
182189
# via feast (setup.py)
183190
frozenlist==1.3.0
@@ -243,7 +250,7 @@ google-resumable-media==1.3.3
243250
# via
244251
# google-cloud-bigquery
245252
# google-cloud-storage
246-
googleapis-common-protos==1.56.2
253+
googleapis-common-protos==1.56.3
247254
# via
248255
# feast (setup.py)
249256
# google-api-core
@@ -252,15 +259,15 @@ great-expectations==0.14.13
252259
# via feast (setup.py)
253260
greenlet==1.1.2
254261
# via sqlalchemy
255-
grpcio==1.46.3
262+
grpcio==1.47.0
256263
# via
257264
# feast (setup.py)
258265
# google-api-core
259266
# google-cloud-bigquery
260267
# grpcio-reflection
261268
# grpcio-testing
262269
# grpcio-tools
263-
grpcio-reflection==1.46.3
270+
grpcio-reflection==1.47.0
264271
# via feast (setup.py)
265272
grpcio-testing==1.44.0
266273
# via feast (setup.py)
@@ -341,7 +348,9 @@ mmh3==3.0.0
341348
# via feast (setup.py)
342349
mock==2.0.0
343350
# via feast (setup.py)
344-
moto==3.1.13
351+
moreorless==0.4.0
352+
# via bowler
353+
moto==3.1.14
345354
# via feast (setup.py)
346355
msal==1.18.0
347356
# via
@@ -369,7 +378,7 @@ mypy-extensions==0.4.3
369378
# via mypy
370379
mypy-protobuf==3.1
371380
# via feast (setup.py)
372-
mysqlclient==2.1.0
381+
mysqlclient==2.1.1
373382
# via feast (setup.py)
374383
nbformat==5.4.0
375384
# via great-expectations
@@ -399,7 +408,7 @@ packaging==21.3
399408
# pytest
400409
# redis
401410
# sphinx
402-
pandas==1.4.2
411+
pandas==1.4.3
403412
# via
404413
# altair
405414
# feast (setup.py)
@@ -492,7 +501,7 @@ pycodestyle==2.8.0
492501
# via flake8
493502
pycparser==2.21
494503
# via cffi
495-
pycryptodomex==3.14.1
504+
pycryptodomex==3.15.0
496505
# via snowflake-connector-python
497506
pydantic==1.9.1
498507
# via
@@ -656,19 +665,19 @@ sphinxcontrib-qthelp==1.0.3
656665
# via sphinx
657666
sphinxcontrib-serializinghtml==1.1.5
658667
# via sphinx
659-
sqlalchemy[mypy]==1.4.37
668+
sqlalchemy[mypy]==1.4.38
660669
# via feast (setup.py)
661670
sqlalchemy2-stubs==0.0.2a24
662671
# via sqlalchemy
663672
stack-data==0.3.0
664673
# via ipython
665674
starlette==0.19.1
666675
# via fastapi
667-
tabulate==0.8.9
676+
tabulate==0.8.10
668677
# via feast (setup.py)
669678
tenacity==8.0.1
670679
# via feast (setup.py)
671-
tensorflow-metadata==1.8.0
680+
tensorflow-metadata==1.9.0
672681
# via feast (setup.py)
673682
termcolor==1.1.0
674683
# via great-expectations
@@ -713,19 +722,19 @@ types-protobuf==3.19.22
713722
# via
714723
# feast (setup.py)
715724
# mypy-protobuf
716-
types-python-dateutil==2.8.17
725+
types-python-dateutil==2.8.18
717726
# via feast (setup.py)
718-
types-pytz==2021.3.8
727+
types-pytz==2022.1.0
719728
# via feast (setup.py)
720729
types-pyyaml==6.0.8
721730
# via feast (setup.py)
722-
types-redis==4.2.7
731+
types-redis==4.3.2
723732
# via feast (setup.py)
724-
types-requests==2.27.30
733+
types-requests==2.27.31
725734
# via feast (setup.py)
726735
types-setuptools==57.4.17
727736
# via feast (setup.py)
728-
types-tabulate==0.8.9
737+
types-tabulate==0.8.10
729738
# via feast (setup.py)
730739
types-urllib3==1.26.15
731740
# via types-requests
@@ -750,17 +759,19 @@ urllib3==1.26.9
750759
# minio
751760
# requests
752761
# responses
753-
uvicorn[standard]==0.17.6
762+
uvicorn[standard]==0.18.1
754763
# via feast (setup.py)
755764
uvloop==0.16.0
756765
# via uvicorn
757766
virtualenv==20.14.1
758767
# via pre-commit
759-
watchgod==0.8.2
768+
volatile==2.1.0
769+
# via bowler
770+
watchfiles==0.15.0
760771
# via uvicorn
761772
wcwidth==0.2.5
762773
# via prompt-toolkit
763-
websocket-client==1.3.2
774+
websocket-client==1.3.3
764775
# via docker
765776
websockets==10.3
766777
# via uvicorn

0 commit comments

Comments
 (0)