From 504e40eae8fd2e2fe9eb4faac61f79bf0a189ff0 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 16 Apr 2024 15:55:25 +0400 Subject: [PATCH 01/16] chore: Upgrade dask dependency (#4050) * add dataframe extra to dask Signed-off-by: tokoko * update pinned versions Signed-off-by: tokoko * add ignore rules to mssql, it's too broken Signed-off-by: tokoko * ruff reformat Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .../contrib/mssql_offline_store/mssql.py | 2 +- .../mssql_offline_store/tests/data_source.py | 2 +- .../requirements/py3.10-ci-requirements.txt | 88 ++++++++++--------- .../requirements/py3.10-requirements.txt | 36 +++++--- .../requirements/py3.9-ci-requirements.txt | 85 ++++++++++-------- .../requirements/py3.9-requirements.txt | 36 +++++--- setup.py | 3 +- 7 files changed, 141 insertions(+), 111 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py index 67bae292c37..f7f42e196b2 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py @@ -423,7 +423,7 @@ def _upload_entity_df_into_sqlserver_and_get_entity_schema( elif isinstance(entity_df, pandas.DataFrame): # Drop the index so that we don't have unnecessary columns - engine.execute(_df_to_create_table_sql(entity_df, table_id)) + engine.execute(_df_to_create_table_sql(entity_df, table_id)) # type: ignore entity_df.to_sql(name=table_id, con=engine, index=False, if_exists="append") entity_schema = dict(zip(entity_df.columns, entity_df.dtypes)), table_id diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py index 71ce56bdefd..bf892e9d969 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py @@ -83,7 +83,7 @@ def create_data_source( engine = create_engine(connection_string) destination_name = self.get_prefixed_table_name(destination_name) # Create table - engine.execute(_df_to_create_table_sql(df, destination_name)) + engine.execute(_df_to_create_table_sql(df, destination_name)) # type: ignore # Upload dataframe to azure table df.to_sql(destination_name, engine, index=False, if_exists="append") diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 235c4dfb72e..b2d9f4f5ba0 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -36,7 +34,7 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -atpublic==4.0 +atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via @@ -61,18 +59,18 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.69 +boto3==1.34.80 # via # feast (setup.py) # moto -botocore==1.34.69 +botocore==1.34.80 # via # boto3 # moto # s3transfer bowler==0.9.0 # via feast (setup.py) -build==1.1.1 +build==1.2.1 # via # feast (setup.py) # pip-tools @@ -138,8 +136,12 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.1 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.10 + # via dask db-dtypes==1.2.0 # via google-cloud-bigquery debugpy==1.8.1 @@ -167,7 +169,7 @@ duckdb==0.10.1 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.2 +duckdb-engine==0.11.4 # via ibis-framework entrypoints==0.4 # via altair @@ -176,15 +178,15 @@ exceptiongroup==1.2.0 # anyio # ipython # pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.0 +fastapi==0.110.1 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat -filelock==3.13.1 +filelock==3.13.3 # via # snowflake-connector-python # virtualenv @@ -214,7 +216,7 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.123.0 +google-api-python-client==2.125.0 # via firebase-admin google-auth==2.29.0 # via @@ -222,6 +224,7 @@ google-auth==2.29.0 # google-api-python-client # google-auth-httplib2 # google-cloud-core + # google-cloud-firestore # google-cloud-storage # kubernetes google-auth-httplib2==0.2.0 @@ -241,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.15.0 +google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 # via @@ -289,7 +292,7 @@ grpcio-testing==1.62.1 # via feast (setup.py) grpcio-tools==1.62.1 # via feast (setup.py) -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via @@ -301,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httplib2==0.22.0 # via @@ -338,9 +341,9 @@ importlib-resources==6.4.0 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.29.3 +ipykernel==6.29.4 # via jupyterlab -ipython==8.22.2 +ipython==8.23.0 # via # great-expectations # ipykernel @@ -413,11 +416,11 @@ jupyter-server==2.13.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.5 +jupyterlab==4.1.6 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.4 +jupyterlab-server==2.26.0 # via # jupyterlab # notebook @@ -482,7 +485,7 @@ nbclient==0.10.0 # via nbconvert nbconvert==7.16.3 # via jupyter-server -nbformat==5.10.3 +nbformat==5.10.4 # via # great-expectations # jupyter-server @@ -501,6 +504,7 @@ notebook-shim==0.2.4 numpy==1.24.4 # via # altair + # dask # db-dtypes # feast (setup.py) # great-expectations @@ -512,13 +516,14 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==21.3 # via # build # dask # db-dtypes # deprecation # docker + # duckdb-engine # google-cloud-bigquery # great-expectations # gunicorn @@ -536,6 +541,8 @@ packaging==24.0 pandas==2.2.1 # via # altair + # dask + # dask-expr # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -544,7 +551,7 @@ pandas==2.2.1 # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert -parso==0.8.3 +parso==0.8.4 # via jedi parsy==2.1 # via ibis-framework @@ -620,6 +627,7 @@ py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # dask-expr # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -627,15 +635,15 @@ pyarrow==15.0.2 # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) -pycparser==2.21 +pycparser==2.22 # via cffi pydantic==2.6.4 # via @@ -655,7 +663,7 @@ pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.11 +pymssql==2.3.0 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) @@ -667,6 +675,7 @@ pyparsing==3.1.2 # via # great-expectations # httplib2 + # packaging pyproject-hooks==1.0.0 # via # build @@ -686,7 +695,7 @@ pytest==7.4.4 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via feast (setup.py) pytest-env==1.1.3 # via feast (setup.py) @@ -791,13 +800,13 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.3.4 +ruff==0.3.5 # via feast (setup.py) s3transfer==0.10.1 # via boto3 -scipy==1.12.0 +scipy==1.13.0 # via great-expectations -send2trash==1.8.2 +send2trash==1.8.3 # via jupyter-server six==1.16.0 # via @@ -838,7 +847,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) @@ -847,15 +856,13 @@ sqlalchemy[mypy]==1.4.52 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 # via ipython -starlette==0.36.3 +starlette==0.37.2 # via fastapi -substrait==0.14.1 +substrait==0.15.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -938,7 +945,7 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.20240311 # via feast (setup.py) -types-redis==4.6.0.20240311 +types-redis==4.6.0.20240409 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) @@ -948,7 +955,7 @@ types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # anyio # async-lru @@ -957,11 +964,12 @@ typing-extensions==4.10.0 # fastapi # great-expectations # ibis-framework + # ipython # mypy # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy # typeguard # uvicorn tzdata==2024.1 @@ -1011,7 +1019,7 @@ websocket-client==1.7.0 # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.1 +werkzeug==3.0.2 # via moto wheel==0.43.0 # via pip-tools diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index aa55cffd5f5..1ac3aca8dbe 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -34,13 +34,17 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.1 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.10 + # via dask dill==0.3.8 # via feast (setup.py) exceptiongroup==1.2.0 # via anyio -fastapi==0.110.0 +fastapi==0.110.1 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -48,7 +52,7 @@ fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via uvicorn @@ -82,10 +86,11 @@ mypy==1.9.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -mypy-protobuf==3.5.0 +mypy-protobuf==3.6.0 # via feast (setup.py) numpy==1.24.4 # via + # dask # feast (setup.py) # pandas # pyarrow @@ -94,7 +99,10 @@ packaging==24.0 # dask # gunicorn pandas==2.2.1 - # via feast (setup.py) + # via + # dask + # dask-expr + # feast (setup.py) partd==1.4.1 # via dask protobuf==4.25.3 @@ -102,7 +110,9 @@ protobuf==4.25.3 # feast (setup.py) # mypy-protobuf pyarrow==15.0.2 - # via feast (setup.py) + # via + # dask-expr + # feast (setup.py) pydantic==2.6.4 # via # fastapi @@ -136,13 +146,11 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -starlette==0.36.3 +starlette==0.37.2 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -160,16 +168,16 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.24.0.20240311 +types-protobuf==4.24.0.20240408 # via mypy-protobuf -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy # typeguard # uvicorn tzdata==2024.1 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 717e82361ab..722e0c5d873 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -36,7 +34,7 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -atpublic==4.0 +atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via @@ -61,18 +59,18 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.69 +boto3==1.34.80 # via # feast (setup.py) # moto -botocore==1.34.69 +botocore==1.34.80 # via # boto3 # moto # s3transfer bowler==0.9.0 # via feast (setup.py) -build==1.1.1 +build==1.2.1 # via # feast (setup.py) # pip-tools @@ -138,8 +136,12 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.1 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.10 + # via dask db-dtypes==1.2.0 # via google-cloud-bigquery debugpy==1.8.1 @@ -167,7 +169,7 @@ duckdb==0.10.1 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.2 +duckdb-engine==0.11.4 # via ibis-framework entrypoints==0.4 # via altair @@ -176,15 +178,15 @@ exceptiongroup==1.2.0 # anyio # ipython # pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.0 +fastapi==0.110.1 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat -filelock==3.13.1 +filelock==3.13.3 # via # snowflake-connector-python # virtualenv @@ -214,7 +216,7 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.123.0 +google-api-python-client==2.125.0 # via firebase-admin google-auth==2.29.0 # via @@ -222,6 +224,7 @@ google-auth==2.29.0 # google-api-python-client # google-auth-httplib2 # google-cloud-core + # google-cloud-firestore # google-cloud-storage # kubernetes google-auth-httplib2==0.2.0 @@ -241,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.15.0 +google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 # via @@ -289,7 +292,7 @@ grpcio-testing==1.62.1 # via feast (setup.py) grpcio-tools==1.62.1 # via feast (setup.py) -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via @@ -301,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httplib2==0.22.0 # via @@ -346,7 +349,7 @@ importlib-resources==6.4.0 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.29.3 +ipykernel==6.29.4 # via jupyterlab ipython==8.18.1 # via @@ -421,11 +424,11 @@ jupyter-server==2.13.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.5 +jupyterlab==4.1.6 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.4 +jupyterlab-server==2.26.0 # via # jupyterlab # notebook @@ -490,7 +493,7 @@ nbclient==0.10.0 # via nbconvert nbconvert==7.16.3 # via jupyter-server -nbformat==5.10.3 +nbformat==5.10.4 # via # great-expectations # jupyter-server @@ -509,6 +512,7 @@ notebook-shim==0.2.4 numpy==1.24.4 # via # altair + # dask # db-dtypes # feast (setup.py) # great-expectations @@ -520,13 +524,14 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==21.3 # via # build # dask # db-dtypes # deprecation # docker + # duckdb-engine # google-cloud-bigquery # great-expectations # gunicorn @@ -544,6 +549,8 @@ packaging==24.0 pandas==2.2.1 # via # altair + # dask + # dask-expr # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -552,7 +559,7 @@ pandas==2.2.1 # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert -parso==0.8.3 +parso==0.8.4 # via jedi parsy==2.1 # via ibis-framework @@ -628,6 +635,7 @@ py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # dask-expr # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -635,15 +643,15 @@ pyarrow==15.0.2 # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) -pycparser==2.21 +pycparser==2.22 # via cffi pydantic==2.6.4 # via @@ -663,7 +671,7 @@ pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.11 +pymssql==2.3.0 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) @@ -675,6 +683,7 @@ pyparsing==3.1.2 # via # great-expectations # httplib2 + # packaging pyproject-hooks==1.0.0 # via # build @@ -694,7 +703,7 @@ pytest==7.4.4 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via feast (setup.py) pytest-env==1.1.3 # via feast (setup.py) @@ -801,13 +810,13 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.3.3 +ruff==0.3.5 # via feast (setup.py) s3transfer==0.10.1 # via boto3 -scipy==1.12.0 +scipy==1.13.0 # via great-expectations -send2trash==1.8.2 +send2trash==1.8.3 # via jupyter-server six==1.16.0 # via @@ -848,7 +857,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) @@ -857,15 +866,13 @@ sqlalchemy[mypy]==1.4.52 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 # via ipython -starlette==0.36.3 +starlette==0.37.2 # via fastapi -substrait==0.14.1 +substrait==0.15.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -948,7 +955,7 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.20240311 # via feast (setup.py) -types-redis==4.6.0.20240311 +types-redis==4.6.0.20240409 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) @@ -958,7 +965,7 @@ types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # anyio # async-lru @@ -972,7 +979,7 @@ typing-extensions==4.10.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy # starlette # typeguard # uvicorn @@ -1024,7 +1031,7 @@ websocket-client==1.7.0 # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.1 +werkzeug==3.0.2 # via moto wheel==0.43.0 # via pip-tools diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 5d1b6814054..ebf00789aec 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -34,13 +34,17 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.1 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.10 + # via dask dill==0.3.8 # via feast (setup.py) exceptiongroup==1.2.0 # via anyio -fastapi==0.110.0 +fastapi==0.110.1 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -48,7 +52,7 @@ fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via uvicorn @@ -83,10 +87,11 @@ mypy==1.9.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -mypy-protobuf==3.5.0 +mypy-protobuf==3.6.0 # via feast (setup.py) numpy==1.24.4 # via + # dask # feast (setup.py) # pandas # pyarrow @@ -95,7 +100,10 @@ packaging==24.0 # dask # gunicorn pandas==2.2.1 - # via feast (setup.py) + # via + # dask + # dask-expr + # feast (setup.py) partd==1.4.1 # via dask protobuf==4.25.3 @@ -103,7 +111,9 @@ protobuf==4.25.3 # feast (setup.py) # mypy-protobuf pyarrow==15.0.2 - # via feast (setup.py) + # via + # dask-expr + # feast (setup.py) pydantic==2.6.4 # via # fastapi @@ -137,13 +147,11 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -starlette==0.36.3 +starlette==0.37.2 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -161,16 +169,16 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.24.0.20240311 +types-protobuf==4.24.0.20240408 # via mypy-protobuf -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy # starlette # typeguard # uvicorn diff --git a/setup.py b/setup.py index c1d872a99f8..af4d58a2782 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - # https://github.com/dask/dask/issues/10996 - "dask>=2021.1.0,<2024.3.0", + "dask[dataframe]>=2021.1.0", "bowler", # Needed for automatic repo upgrades "importlib-resources>=6.0.0,<7", "importlib_metadata>=6.8.0,<7", From 5c4a9c57fa42ee5688fb6b428cedb416a7dbf185 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Tue, 16 Apr 2024 11:38:14 -0700 Subject: [PATCH 02/16] fix: Pgvector patch (#4103) --- Makefile | 19 +++++++ docs/reference/online-stores/postgres.md | 28 ++++++++++ sdk/python/feast/infra/key_encoding_utils.py | 9 +++- .../contrib/pgvector_repo_configuration.py | 12 +++++ .../infra/online_stores/contrib/postgres.py | 51 +++++++++++-------- .../contrib/postgres_repo_configuration.py | 6 --- .../universal/online_store/init.sql | 1 + .../universal/online_store/postgres.py | 12 ++++- 8 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 sdk/python/feast/infra/online_stores/contrib/pgvector_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/init.sql diff --git a/Makefile b/Makefile index 6fcf95dc7da..bf2d876b7f1 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,25 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests + test-python-universal-pgvector-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.pgvector_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.postgres \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal-mysql-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 3885867dd26..277494868c8 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -30,6 +30,8 @@ online_store: sslkey_path: /path/to/client-key.pem sslcert_path: /path/to/client-cert.pem sslrootcert_path: /path/to/server-ca.pem + pgvector_enabled: false + vector_len: 512 ``` {% endcode %} @@ -60,3 +62,29 @@ Below is a matrix indicating which functionality is supported by the Postgres on | collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## PGVector +The Postgres online store supports the use of [PGVector](https://pgvector.dev/) for storing feature values. +To enable PGVector, set `pgvector_enabled: true` in the online store configuration. +The `vector_len` parameter can be used to specify the length of the vector. The default value is 512. + +Then you can use `retrieve_online_documents` to retrieve the top k closest vectors to a query vector. + +{% code title="python" %} +```python +from feast import FeatureStore +from feast.infra.online_stores.postgres import retrieve_online_documents + +feature_store = FeatureStore(repo_path=".") + +query_vector = [0.1, 0.2, 0.3, 0.4, 0.5] +top_k = 5 + +feature_values = retrieve_online_documents( + feature_store=feature_store, + feature_view_name="document_fv:embedding_float", + query_vector=query_vector, + top_k=top_k, +) +``` +{% endcode %} diff --git a/sdk/python/feast/infra/key_encoding_utils.py b/sdk/python/feast/infra/key_encoding_utils.py index e50e438c3de..ca834f19176 100644 --- a/sdk/python/feast/infra/key_encoding_utils.py +++ b/sdk/python/feast/infra/key_encoding_utils.py @@ -74,8 +74,13 @@ def serialize_entity_key( return b"".join(output) -def get_val_str(val): - accept_value_types = ["float_list_val", "double_list_val", "int_list_val"] +def get_list_val_str(val): + accept_value_types = [ + "float_list_val", + "double_list_val", + "int32_list_val", + "int64_list_val", + ] for accept_type in accept_value_types: if val.HasField(accept_type): return str(getattr(val, accept_type).val) diff --git a/sdk/python/feast/infra/online_stores/contrib/pgvector_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/pgvector_repo_configuration.py new file mode 100644 index 00000000000..26b05613158 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/pgvector_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.postgres import ( + PGVectorOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="pgvector", online_store_creator=PGVectorOnlineStoreCreator + ), +] diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 2dcb6187837..2890f60746b 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,7 +2,7 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import psycopg2 import pytz @@ -12,7 +12,7 @@ from feast import Entity from feast.feature_view import FeatureView -from feast.infra.key_encoding_utils import get_val_str, serialize_entity_key +from feast.infra.key_encoding_utils import get_list_val_str, serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.infra.utils.postgres.connection_utils import _get_conn, _get_connection_pool from feast.infra.utils.postgres.postgres_config import ConnectionType, PostgreSQLConfig @@ -74,19 +74,18 @@ def online_write_batch( created_ts = _to_naive_utc(created_ts) for feature_name, val in values.items(): - val_str: Union[str, bytes] + vector_val = None if ( - "pgvector_enabled" in config.online_config - and config.online_config["pgvector_enabled"] + "pgvector_enabled" in config.online_store + and config.online_store.pgvector_enabled ): - val_str = get_val_str(val) - else: - val_str = val.SerializeToString() + vector_val = get_list_val_str(val) insert_values.append( ( entity_key_bin, feature_name, - val_str, + val.SerializeToString(), + vector_val, timestamp, created_ts, ) @@ -100,11 +99,12 @@ def online_write_batch( sql.SQL( """ INSERT INTO {} - (entity_key, feature_name, value, event_ts, created_ts) + (entity_key, feature_name, value, vector_value, event_ts, created_ts) VALUES %s ON CONFLICT (entity_key, feature_name) DO UPDATE SET value = EXCLUDED.value, + vector_value = EXCLUDED.vector_value, event_ts = EXCLUDED.event_ts, created_ts = EXCLUDED.created_ts; """, @@ -226,12 +226,14 @@ def update( for table in tables_to_keep: table_name = _table_id(project, table) - value_type = "BYTEA" if ( - "pgvector_enabled" in config.online_config - and config.online_config["pgvector_enabled"] + "pgvector_enabled" in config.online_store + and config.online_store.pgvector_enabled ): - value_type = f'vector({config.online_config["vector_len"]})' + vector_value_type = f"vector({config.online_store.vector_len})" + else: + # keep the vector_value_type as BYTEA if pgvector is not enabled, to maintain compatibility + vector_value_type = "BYTEA" cur.execute( sql.SQL( """ @@ -239,7 +241,8 @@ def update( ( entity_key BYTEA, feature_name TEXT, - value {}, + value BYTEA, + vector_value {} NULL, event_ts TIMESTAMPTZ, created_ts TIMESTAMPTZ, PRIMARY KEY(entity_key, feature_name) @@ -248,7 +251,7 @@ def update( """ ).format( sql.Identifier(table_name), - sql.SQL(value_type), + sql.SQL(vector_value_type), sql.Identifier(f"{table_name}_ek"), sql.Identifier(table_name), ) @@ -294,6 +297,14 @@ def retrieve_online_documents( """ project = config.project + if ( + "pgvector_enabled" not in config.online_store + or not config.online_store.pgvector_enabled + ): + raise ValueError( + "pgvector is not enabled in the online store configuration" + ) + # Convert the embedding to a string to be used in postgres vector search query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" @@ -311,8 +322,8 @@ def retrieve_online_documents( SELECT entity_key, feature_name, - value, - value <-> %s as distance, + vector_value, + vector_value <-> %s as distance, event_ts FROM {table_name} WHERE feature_name = {feature_name} ORDER BY distance @@ -327,13 +338,13 @@ def retrieve_online_documents( ) rows = cur.fetchall() - for entity_key, feature_name, value, distance, event_ts in rows: + for entity_key, feature_name, vector_value, distance, event_ts in rows: # TODO Deserialize entity_key to return the entity in response # entity_key_proto = EntityKeyProto() # entity_key_proto_bin = bytes(entity_key) # TODO Convert to List[float] for value type proto - feature_value_proto = ValueProto(string_val=value) + feature_value_proto = ValueProto(string_val=vector_value) distance_value_proto = ValueProto(float_val=distance) result.append((event_ts, feature_value_proto, distance_value_proto)) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/postgres_repo_configuration.py index 6e4ca3f9501..ea975ec808f 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres_repo_configuration.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres_repo_configuration.py @@ -2,7 +2,6 @@ IntegrationTestRepoConfig, ) from tests.integration.feature_repos.universal.online_store.postgres import ( - PGVectorOnlineStoreCreator, PostgresOnlineStoreCreator, ) @@ -10,9 +9,4 @@ IntegrationTestRepoConfig( online_store="postgres", online_store_creator=PostgresOnlineStoreCreator ), - IntegrationTestRepoConfig( - online_store="pgvector", online_store_creator=PGVectorOnlineStoreCreator - ), ] - -AVAILABLE_ONLINE_STORES = {"pgvector": PGVectorOnlineStoreCreator} diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/init.sql b/sdk/python/tests/integration/feature_repos/universal/online_store/init.sql new file mode 100644 index 00000000000..64f04f61ad3 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/init.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS vector; \ No newline at end of file diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py index 58e7af9c468..7b4156fffe0 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py @@ -1,3 +1,4 @@ +import os from typing import Dict from testcontainers.core.container import DockerContainer @@ -37,12 +38,17 @@ def teardown(self): class PGVectorOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) + script_directory = os.path.dirname(os.path.abspath(__file__)) self.container = ( DockerContainer("pgvector/pgvector:pg16") .with_env("POSTGRES_USER", "root") .with_env("POSTGRES_PASSWORD", "test") .with_env("POSTGRES_DB", "test") .with_exposed_ports(5432) + .with_volume_mapping( + os.path.join(script_directory, "init.sql"), + "/docker-entrypoint-initdb.d/init.sql", + ) ) def create_online_store(self) -> Dict[str, str]: @@ -51,8 +57,10 @@ def create_online_store(self) -> Dict[str, str]: wait_for_logs( container=self.container, predicate=log_string_to_wait_for, timeout=10 ) - command = "psql -h localhost -p 5432 -U root -d test -c 'CREATE EXTENSION IF NOT EXISTS vector;'" - self.container.exec(command) + init_log_string_to_wait_for = "PostgreSQL init process complete" + wait_for_logs( + container=self.container, predicate=init_log_string_to_wait_for, timeout=10 + ) return { "host": "localhost", "type": "postgres", From 4362b6cc857ceafe60d58a14f2dfe006a83effb8 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 17 Apr 2024 09:35:25 +0400 Subject: [PATCH 03/16] fix: Remove top-level grpc import in cli (#4107) fix grpc import in cli Signed-off-by: tokoko --- sdk/python/feast/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 7673eee20db..f23cc62bd81 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -31,7 +31,6 @@ ) from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError from feast.feature_view import FeatureView -from feast.infra.contrib.grpc_server import get_grpc_server from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import load_repo_config from feast.repo_operations import ( @@ -734,6 +733,8 @@ def listen_command( registry_ttl_sec: int, ): """Start a gRPC feature server to ingest streaming features on given address""" + from feast.infra.contrib.grpc_server import get_grpc_server + store = create_feature_store(ctx) server = get_grpc_server(address, store, max_workers, registry_ttl_sec) server.start() From 77c78aad00245294c5372bd3416e8b6bf88ab26b Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 17 Apr 2024 13:12:57 +0700 Subject: [PATCH 04/16] chore: Omit bytewax deps and materialization engine (#4098) --- CODEOWNERS | 3 - .../contrib/bytewax/Dockerfile | 29 - .../contrib/bytewax/__init__.py | 15 - .../bytewax_materialization_dataflow.py | 83 --- .../bytewax/bytewax_materialization_engine.py | 495 ------------------ .../bytewax/bytewax_materialization_job.py | 63 --- .../bytewax/bytewax_materialization_task.py | 10 - .../contrib/bytewax/dataflow.py | 28 - .../contrib/bytewax/entrypoint.sh | 4 - sdk/python/feast/repo_config.py | 1 - .../requirements/py3.10-ci-requirements.txt | 5 - .../requirements/py3.9-ci-requirements.txt | 5 - .../materialization/contrib/bytewax/README.md | 22 - .../contrib/bytewax/eks-config.yaml | 13 - .../contrib/bytewax/test_bytewax.py | 67 --- setup.py | 4 - 16 files changed, 847 deletions(-) delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh delete mode 100644 sdk/python/tests/integration/materialization/contrib/bytewax/README.md delete mode 100644 sdk/python/tests/integration/materialization/contrib/bytewax/eks-config.yaml delete mode 100644 sdk/python/tests/integration/materialization/contrib/bytewax/test_bytewax.py diff --git a/CODEOWNERS b/CODEOWNERS index bb154a71481..18914d9f5dc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -51,8 +51,5 @@ # Snowflake /sdk/python/feast/infra/materialization/snowflake* @sfc-gh-madkins -# Bytewax -/sdk/python/feast/infra/materialization/contrib/bytewax/ @whoahbot - # AWS Lambda /sdk/python/feast/infra/materialization/contrib/aws_lambda/ @achals diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile deleted file mode 100644 index a7d0af9b416..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM python:3.9-slim-bullseye AS build - -RUN apt-get update && \ - apt-get install --no-install-suggests --no-install-recommends --yes git - -WORKDIR /bytewax - -# Copy dataflow code -COPY sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py /bytewax -COPY sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py /bytewax - -# Copy entrypoint -COPY sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh /bytewax - -# Copy necessary parts of the Feast codebase -COPY sdk/python sdk/python -COPY protos protos -COPY go go -COPY setup.py setup.py -COPY pyproject.toml pyproject.toml -COPY README.md README.md - -# Install Feast for AWS with Bytewax dependencies -# We need this mount thingy because setuptools_scm needs access to the -# git dir to infer the version of feast we're installing. -# https://github.com/pypa/setuptools_scm#usage-from-docker -# I think it also assumes that this dockerfile is being built from the root of the directory. -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[aws,gcp,bytewax,snowflake,postgres]' - diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py deleted file mode 100644 index 0838a4c0d59..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .bytewax_materialization_dataflow import BytewaxMaterializationDataflow -from .bytewax_materialization_engine import ( - BytewaxMaterializationEngine, - BytewaxMaterializationEngineConfig, -) -from .bytewax_materialization_job import BytewaxMaterializationJob -from .bytewax_materialization_task import BytewaxMaterializationTask - -__all__ = [ - "BytewaxMaterializationTask", - "BytewaxMaterializationJob", - "BytewaxMaterializationDataflow", - "BytewaxMaterializationEngine", - "BytewaxMaterializationEngineConfig", -] diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py deleted file mode 100644 index 6fc53b67f2f..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -import os -from typing import List - -import pyarrow as pa -import pyarrow.parquet as pq -from bytewax.dataflow import Dataflow # type: ignore -from bytewax.execution import cluster_main -from bytewax.inputs import ManualInputConfig -from bytewax.outputs import ManualOutputConfig - -from feast import FeatureStore, FeatureView, RepoConfig -from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping - -logger = logging.getLogger(__name__) -DEFAULT_BATCH_SIZE = 1000 - - -class BytewaxMaterializationDataflow: - def __init__( - self, - config: RepoConfig, - feature_view: FeatureView, - paths: List[str], - worker_index: int, - ): - self.config = config - self.feature_store = FeatureStore(config=config) - - self.feature_view = feature_view - self.worker_index = worker_index - self.paths = paths - self.mini_batch_size = int( - os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE) - ) - - self._run_dataflow() - - def process_path(self, path): - logger.info(f"Processing path {path}") - dataset = pq.ParquetDataset(path, use_legacy_dataset=False) - batches = [] - for fragment in dataset.fragments: - for batch in fragment.to_table().to_batches( - max_chunksize=self.mini_batch_size - ): - batches.append(batch) - - return batches - - def input_builder(self, worker_index, worker_count, _state): - return [(None, self.paths[self.worker_index])] - - def output_builder(self, worker_index, worker_count): - def output_fn(mini_batch): - table: pa.Table = pa.Table.from_batches([mini_batch]) - - if self.feature_view.batch_source.field_mapping is not None: - table = _run_pyarrow_field_mapping( - table, self.feature_view.batch_source.field_mapping - ) - join_key_to_value_type = { - entity.name: entity.dtype.to_value_type() - for entity in self.feature_view.entity_columns - } - rows_to_write = _convert_arrow_to_proto( - table, self.feature_view, join_key_to_value_type - ) - self.feature_store._get_provider().online_write_batch( - config=self.config, - table=self.feature_view, - data=rows_to_write, - progress=None, - ) - - return output_fn - - def _run_dataflow(self): - flow = Dataflow() - flow.input("inp", ManualInputConfig(self.input_builder)) - flow.flat_map(self.process_path) - flow.capture(ManualOutputConfig(self.output_builder)) - cluster_main(flow, [], 0) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py deleted file mode 100644 index 3ad6fe4b559..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ /dev/null @@ -1,495 +0,0 @@ -import logging -import uuid -from datetime import datetime -from time import sleep -from typing import Callable, List, Literal, Sequence, Union - -import yaml -from kubernetes import client, utils -from kubernetes import config as k8s_config -from kubernetes.client.exceptions import ApiException -from kubernetes.utils import FailToCreateError -from pydantic import StrictStr -from tqdm import tqdm - -from feast import FeatureView, RepoConfig -from feast.batch_feature_view import BatchFeatureView -from feast.entity import Entity -from feast.infra.materialization.batch_materialization_engine import ( - BatchMaterializationEngine, - MaterializationJob, - MaterializationJobStatus, - MaterializationTask, -) -from feast.infra.offline_stores.offline_store import OfflineStore -from feast.infra.online_stores.online_store import OnlineStore -from feast.infra.registry.base_registry import BaseRegistry -from feast.repo_config import FeastConfigBaseModel -from feast.stream_feature_view import StreamFeatureView -from feast.utils import _get_column_names - -from .bytewax_materialization_job import BytewaxMaterializationJob - -logger = logging.getLogger(__name__) - - -class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): - """Batch Materialization Engine config for Bytewax""" - - type: Literal["bytewax"] = "bytewax" - """ Materialization type selector""" - - namespace: StrictStr = "default" - """ (optional) The namespace in Kubernetes to use when creating services, configuration maps and jobs. - """ - - image: StrictStr = "bytewax/bytewax-feast:latest" - """ (optional) The container image to use when running the materialization job.""" - - env: List[dict] = [] - """ (optional) A list of environment variables to set in the created Kubernetes pods. - These environment variables can be used to reference Kubernetes secrets. - """ - - image_pull_secrets: List[dict] = [] - """ (optional) The secrets to use when pulling the image to run for the materialization job """ - - resources: dict = {} - """ (optional) The resource requests and limits for the materialization containers """ - - service_account_name: StrictStr = "" - """ (optional) The service account name to use when running the job """ - - annotations: dict = {} - """ (optional) Annotations to apply to the job container. Useful for linking the service account to IAM roles, operational metadata, etc """ - - include_security_context_capabilities: bool = True - """ (optional) Include security context capabilities in the init and job container spec """ - - labels: dict = {} - """ (optional) additional labels to append to kubernetes objects """ - - max_parallelism: int = 10 - """ (optional) Maximum number of pods allowed to run in parallel""" - - synchronous: bool = False - """ (optional) If true, wait for materialization for one feature to complete before moving to the next """ - - retry_limit: int = 2 - """ (optional) Maximum number of times to retry a materialization worker pod""" - - mini_batch_size: int = 1000 - """ (optional) Number of rows to process per write operation (default 1000)""" - - active_deadline_seconds: int = 86400 - """ (optional) Maximum amount of time a materialization job is allowed to run""" - - job_batch_size: int = 100 - """ (optional) Maximum number of pods to process per job. Only applies to synchronous materialization""" - - print_pod_logs_on_failure: bool = True - """(optional) Print pod logs on job failure. Only applies to synchronous materialization""" - - -class BytewaxMaterializationEngine(BatchMaterializationEngine): - def __init__( - self, - *, - repo_config: RepoConfig, - offline_store: OfflineStore, - online_store: OnlineStore, - **kwargs, - ): - super().__init__( - repo_config=repo_config, - offline_store=offline_store, - online_store=online_store, - **kwargs, - ) - self.repo_config = repo_config - self.offline_store = offline_store - self.online_store = online_store - - k8s_config.load_config() - - self.k8s_client = client.api_client.ApiClient() - self.v1 = client.CoreV1Api(self.k8s_client) - self.batch_v1 = client.BatchV1Api(self.k8s_client) - self.batch_engine_config = repo_config.batch_engine - self.namespace = self.batch_engine_config.namespace - - def update( - self, - project: str, - views_to_delete: Sequence[ - Union[BatchFeatureView, StreamFeatureView, FeatureView] - ], - views_to_keep: Sequence[ - Union[BatchFeatureView, StreamFeatureView, FeatureView] - ], - entities_to_delete: Sequence[Entity], - entities_to_keep: Sequence[Entity], - ): - """This method ensures that any necessary infrastructure or resources needed by the - engine are set up ahead of materialization.""" - pass - - def teardown_infra( - self, - project: str, - fvs: Sequence[Union[BatchFeatureView, StreamFeatureView, FeatureView]], - entities: Sequence[Entity], - ): - """This method ensures that any infrastructure or resources set up by ``update()``are torn down.""" - pass - - def materialize( - self, - registry: BaseRegistry, - tasks: List[MaterializationTask], - ) -> List[MaterializationJob]: - return [ - self._materialize_one( - registry, - task.feature_view, - task.start_time, - task.end_time, - task.project, - task.tqdm_builder, - ) - for task in tasks - ] - - def _materialize_one( - self, - registry: BaseRegistry, - feature_view: Union[BatchFeatureView, StreamFeatureView, FeatureView], - start_date: datetime, - end_date: datetime, - project: str, - tqdm_builder: Callable[[int], tqdm], - ): - entities = [] - for entity_name in feature_view.entities: - entities.append(registry.get_entity(entity_name, project)) - - ( - join_key_columns, - feature_name_columns, - timestamp_field, - created_timestamp_column, - ) = _get_column_names(feature_view, entities) - - offline_job = self.offline_store.pull_latest_from_table_or_query( - config=self.repo_config, - data_source=feature_view.batch_source, - join_key_columns=join_key_columns, - feature_name_columns=feature_name_columns, - timestamp_field=timestamp_field, - created_timestamp_column=created_timestamp_column, - start_date=start_date, - end_date=end_date, - ) - - paths = offline_job.to_remote_storage() - if self.batch_engine_config.synchronous: - offset = 0 - total_pods = len(paths) - batch_size = self.batch_engine_config.job_batch_size - if batch_size < 1: - raise ValueError("job_batch_size must be a value greater than 0") - if batch_size < self.batch_engine_config.max_parallelism: - logger.warning( - "job_batch_size is less than max_parallelism. Setting job_batch_size = max_parallelism" - ) - batch_size = self.batch_engine_config.max_parallelism - - while True: - next_offset = min(offset + batch_size, total_pods) - job = self._await_path_materialization( - paths[offset:next_offset], - feature_view, - offset, - next_offset, - total_pods, - ) - offset += batch_size - if ( - offset >= total_pods - or job.status() == MaterializationJobStatus.ERROR - ): - break - else: - job_id = str(uuid.uuid4()) - job = self._create_kubernetes_job(job_id, paths, feature_view) - - return job - - def _await_path_materialization( - self, paths, feature_view, batch_start, batch_end, total_pods - ): - job_id = str(uuid.uuid4()) - job = self._create_kubernetes_job(job_id, paths, feature_view) - - try: - while job.status() in ( - MaterializationJobStatus.WAITING, - MaterializationJobStatus.RUNNING, - ): - logger.info( - f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " - f"(of {total_pods}) running..." - ) - sleep(30) - logger.info( - f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " - f"(of {total_pods}) complete with status {job.status()}" - ) - except BaseException as e: - logger.info(f"Deleting job {job.job_id()}") - try: - self.batch_v1.delete_namespaced_job(job.job_id(), self.namespace) - except ApiException as ae: - logger.warning(f"Could not delete job due to API Error: {ae.body}") - raise e - finally: - logger.info(f"Deleting configmap {self._configmap_name(job_id)}") - try: - self.v1.delete_namespaced_config_map( - self._configmap_name(job_id), self.namespace - ) - except ApiException as ae: - logger.warning( - f"Could not delete configmap due to API Error: {ae.body}" - ) - - if ( - job.status() == MaterializationJobStatus.ERROR - and self.batch_engine_config.print_pod_logs_on_failure - ): - self._print_pod_logs(job.job_id(), feature_view, batch_start) - - return job - - def _print_pod_logs(self, job_id, feature_view, offset=0): - pods_list = self.v1.list_namespaced_pod( - namespace=self.namespace, - label_selector=f"job-name={job_id}", - ).items - for i, pod in enumerate(pods_list): - logger.info(f"Logging output for {feature_view.name} pod {offset+i}") - try: - logger.info( - self.v1.read_namespaced_pod_log(pod.metadata.name, self.namespace) - ) - except ApiException as e: - logger.warning(f"Could not retrieve pod logs due to: {e.body}") - - def _create_kubernetes_job(self, job_id, paths, feature_view): - try: - # Create a k8s configmap with information needed by bytewax - self._create_configuration_map(job_id, paths, feature_view, self.namespace) - - # Create the k8s job definition - self._create_job_definition( - job_id, - self.namespace, - len(paths), # Create a pod for each parquet file - self.batch_engine_config.env, - ) - logger.info( - f"Created job `dataflow-{job_id}` on namespace `{self.namespace}`" - ) - except FailToCreateError as failures: - return BytewaxMaterializationJob(job_id, self.namespace, error=failures) - - return BytewaxMaterializationJob(job_id, self.namespace) - - def _create_configuration_map(self, job_id, paths, feature_view, namespace): - """Create a Kubernetes configmap for this job""" - - feature_store_configuration = yaml.dump(self.repo_config.dict(by_alias=True)) - - materialization_config = yaml.dump( - {"paths": paths, "feature_view": feature_view.name} - ) - - labels = {"feast-bytewax-materializer": "configmap"} - configmap_manifest = { - "kind": "ConfigMap", - "apiVersion": "v1", - "metadata": { - "name": self._configmap_name(job_id), - "labels": {**labels, **self.batch_engine_config.labels}, - }, - "data": { - "feature_store.yaml": feature_store_configuration, - "bytewax_materialization_config.yaml": materialization_config, - }, - } - self.v1.create_namespaced_config_map( - namespace=namespace, - body=configmap_manifest, - ) - - def _configmap_name(self, job_id): - return f"feast-{job_id}" - - def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0): - """Create a kubernetes job definition.""" - job_env = [ - {"name": "RUST_BACKTRACE", "value": "full"}, - { - "name": "BYTEWAX_PYTHON_FILE_PATH", - "value": "/bytewax/dataflow.py", - }, - {"name": "BYTEWAX_WORKDIR", "value": "/bytewax"}, - { - "name": "BYTEWAX_WORKERS_PER_PROCESS", - "value": "1", - }, - { - "name": "BYTEWAX_POD_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.annotations['batch.kubernetes.io/job-completion-index']", - } - }, - }, - { - "name": "BYTEWAX_REPLICAS", - "value": "1", - }, - { - "name": "BYTEWAX_KEEP_CONTAINER_ALIVE", - "value": "false", - }, - { - "name": "BYTEWAX_STATEFULSET_NAME", - "value": f"dataflow-{job_id}", - }, - { - "name": "BYTEWAX_MINI_BATCH_SIZE", - "value": str(self.batch_engine_config.mini_batch_size), - }, - ] - # Add any Feast configured environment variables - job_env.extend(env) - - securityContextCapabilities = None - if self.batch_engine_config.include_security_context_capabilities: - securityContextCapabilities = { - "add": ["NET_BIND_SERVICE"], - "drop": ["ALL"], - } - - job_labels = {"feast-bytewax-materializer": "job"} - pod_labels = {"feast-bytewax-materializer": "pod"} - job_definition = { - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": f"dataflow-{job_id}", - "namespace": namespace, - "labels": {**job_labels, **self.batch_engine_config.labels}, - }, - "spec": { - "ttlSecondsAfterFinished": 3600, - "backoffLimit": self.batch_engine_config.retry_limit, - "completions": pods, - "parallelism": min(pods, self.batch_engine_config.max_parallelism), - "activeDeadlineSeconds": self.batch_engine_config.active_deadline_seconds, - "completionMode": "Indexed", - "template": { - "metadata": { - "annotations": self.batch_engine_config.annotations, - "labels": {**pod_labels, **self.batch_engine_config.labels}, - }, - "spec": { - "restartPolicy": "Never", - "subdomain": f"dataflow-{job_id}", - "imagePullSecrets": self.batch_engine_config.image_pull_secrets, - "serviceAccountName": self.batch_engine_config.service_account_name, - "initContainers": [ - { - "env": [ - { - "name": "BYTEWAX_REPLICAS", - "value": f"{pods}", - } - ], - "image": "busybox", - "imagePullPolicy": "IfNotPresent", - "name": "init-hostfile", - "resources": {}, - "securityContext": { - "allowPrivilegeEscalation": False, - "capabilities": securityContextCapabilities, - "readOnlyRootFilesystem": True, - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - {"mountPath": "/etc/bytewax", "name": "hostfile"}, - { - "mountPath": "/tmp/bytewax/", - "name": "python-files", - }, - { - "mountPath": "/var/feast/", - "name": self._configmap_name(job_id), - }, - ], - } - ], - "containers": [ - { - "command": ["sh", "-c", "sh ./entrypoint.sh"], - "env": job_env, - "image": self.batch_engine_config.image, - "imagePullPolicy": "Always", - "name": "process", - "ports": [ - { - "containerPort": 9999, - "name": "process", - "protocol": "TCP", - } - ], - "resources": self.batch_engine_config.resources, - "securityContext": { - "allowPrivilegeEscalation": False, - "capabilities": securityContextCapabilities, - "readOnlyRootFilesystem": False, - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - {"mountPath": "/etc/bytewax", "name": "hostfile"}, - { - "mountPath": "/var/feast/", - "name": self._configmap_name(job_id), - }, - ], - } - ], - "volumes": [ - {"emptyDir": {}, "name": "hostfile"}, - { - "configMap": { - "defaultMode": 420, - "name": self._configmap_name(job_id), - }, - "name": "python-files", - }, - { - "configMap": {"name": self._configmap_name(job_id)}, - "name": self._configmap_name(job_id), - }, - ], - }, - }, - }, - } - utils.create_from_dict(self.k8s_client, job_definition) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py deleted file mode 100644 index da969d5a880..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import Optional - -from kubernetes import client - -from feast.infra.materialization.batch_materialization_engine import ( - MaterializationJob, - MaterializationJobStatus, -) - - -class BytewaxMaterializationJob(MaterializationJob): - def __init__( - self, - job_id, - namespace, - error: Optional[BaseException] = None, - ): - super().__init__() - self._job_id = job_id - self.namespace = namespace - self._error: Optional[BaseException] = error - self.batch_v1 = client.BatchV1Api() - - def error(self): - return self._error - - def status(self): - if self._error is not None: - return MaterializationJobStatus.ERROR - else: - # TODO: Find a better way to parse status? - job_status = self.batch_v1.read_namespaced_job_status( - self.job_id(), self.namespace - ).status - if job_status.active is not None: - if job_status.completion_time is None: - return MaterializationJobStatus.RUNNING - else: - if ( - job_status.completion_time is not None - and job_status.conditions[0].type == "Complete" - ): - return MaterializationJobStatus.SUCCEEDED - - if ( - job_status.conditions is not None - and job_status.conditions[0].type == "Failed" - ): - self._error = Exception( - f"Job {self.job_id()} failed with reason: " - f"{job_status.conditions[0].message}" - ) - return MaterializationJobStatus.ERROR - return MaterializationJobStatus.WAITING - - def should_be_retried(self): - return False - - def job_id(self): - return f"dataflow-{self._job_id}" - - def url(self): - return None diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py deleted file mode 100644 index 8bb8da741aa..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py +++ /dev/null @@ -1,10 +0,0 @@ -from feast.infra.materialization.batch_materialization_engine import MaterializationTask - - -class BytewaxMaterializationTask(MaterializationTask): - def __init__(self, project, feature_view, start_date, end_date, tqdm): - self.project = project - self.feature_view = feature_view - self.start_date = start_date - self.end_date = end_date - self.tqdm = tqdm diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py deleted file mode 100644 index bbc32cc1651..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -import os - -import yaml - -from feast import FeatureStore, RepoConfig -from feast.infra.materialization.contrib.bytewax.bytewax_materialization_dataflow import ( - BytewaxMaterializationDataflow, -) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - with open("/var/feast/feature_store.yaml") as f: - feast_config = yaml.load(f, Loader=yaml.Loader) - - with open("/var/feast/bytewax_materialization_config.yaml") as b: - bytewax_config = yaml.load(b, Loader=yaml.Loader) - - config = RepoConfig(**feast_config) - store = FeatureStore(config=config) - - job = BytewaxMaterializationDataflow( - config, - store.get_feature_view(bytewax_config["feature_view"]), - bytewax_config["paths"], - int(os.environ["JOB_COMPLETION_INDEX"]), - ) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh b/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh deleted file mode 100644 index 0179e5481fa..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd /bytewax -python dataflow.py diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 26ada13fc59..e8185f4a4ab 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -46,7 +46,6 @@ "local": "feast.infra.materialization.local_engine.LocalMaterializationEngine", "snowflake.engine": "feast.infra.materialization.snowflake_engine.SnowflakeMaterializationEngine", "lambda": "feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationEngine", - "bytewax": "feast.infra.materialization.contrib.bytewax.bytewax_materialization_engine.BytewaxMaterializationEngine", "k8s": "feast.infra.materialization.kubernetes.kubernetes_materialization_engine.KubernetesMaterializationEngine", "spark.engine": "feast.infra.materialization.contrib.spark.spark_materialization_engine.SparkMaterializationEngine", } diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index b2d9f4f5ba0..ab6a670f32b 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -74,8 +74,6 @@ build==1.2.1 # via # feast (setup.py) # pip-tools -bytewax==0.15.1 - # via feast (setup.py) cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 @@ -154,7 +152,6 @@ deprecation==2.1.0 # via testcontainers dill==0.3.8 # via - # bytewax # feast (setup.py) # multiprocess distlib==0.3.8 @@ -471,8 +468,6 @@ msgpack==1.0.8 # via cachecontrol multipledispatch==1.0.0 # via ibis-framework -multiprocess==0.70.16 - # via bytewax mypy==1.9.0 # via # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 722e0c5d873..9178b1de127 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -74,8 +74,6 @@ build==1.2.1 # via # feast (setup.py) # pip-tools -bytewax==0.15.1 - # via feast (setup.py) cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 @@ -154,7 +152,6 @@ deprecation==2.1.0 # via testcontainers dill==0.3.8 # via - # bytewax # feast (setup.py) # multiprocess distlib==0.3.8 @@ -479,8 +476,6 @@ msgpack==1.0.8 # via cachecontrol multipledispatch==1.0.0 # via ibis-framework -multiprocess==0.70.16 - # via bytewax mypy==1.9.0 # via # feast (setup.py) diff --git a/sdk/python/tests/integration/materialization/contrib/bytewax/README.md b/sdk/python/tests/integration/materialization/contrib/bytewax/README.md deleted file mode 100644 index 4ed5d49a680..00000000000 --- a/sdk/python/tests/integration/materialization/contrib/bytewax/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Running Bytewax integration tests - -To run the Bytewax integration tests, you'll need to provision a cluster using [eksctl.](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html). - -## Creating an EKS cluster - -In this directory is a configuration file for a single-node EKS cluster - -To create the EKS cluster needed for testing, issue the following command: - -``` shell -> eksctl create cluster -f ./eks-config.yaml -``` - -When the tests are complete, delete the created cluster with: - -``` shell -> eksctl delete cluster bytewax-feast-cluster -``` - - - diff --git a/sdk/python/tests/integration/materialization/contrib/bytewax/eks-config.yaml b/sdk/python/tests/integration/materialization/contrib/bytewax/eks-config.yaml deleted file mode 100644 index 5f8d0655aac..00000000000 --- a/sdk/python/tests/integration/materialization/contrib/bytewax/eks-config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: bytewax-feast-cluster - version: "1.22" - region: us-west-2 - -managedNodeGroups: -- name: ng-1 - instanceType: c6a.large - desiredCapacity: 1 - privateNetworking: true diff --git a/sdk/python/tests/integration/materialization/contrib/bytewax/test_bytewax.py b/sdk/python/tests/integration/materialization/contrib/bytewax/test_bytewax.py deleted file mode 100644 index 0d2cecb2f14..00000000000 --- a/sdk/python/tests/integration/materialization/contrib/bytewax/test_bytewax.py +++ /dev/null @@ -1,67 +0,0 @@ -from datetime import timedelta - -import pytest - -from feast import Entity, Feature, FeatureView, ValueType -from tests.data.data_creator import create_basic_driver_dataset -from tests.integration.feature_repos.integration_test_repo_config import ( - IntegrationTestRepoConfig, - RegistryLocation, -) -from tests.integration.feature_repos.repo_configuration import ( - construct_test_environment, -) -from tests.integration.feature_repos.universal.data_sources.redshift import ( - RedshiftDataSourceCreator, -) -from tests.utils.e2e_test_validation import validate_offline_online_store_consistency - - -@pytest.mark.integration -@pytest.mark.skip(reason="Run this test manually after creating an EKS cluster.") -def test_bytewax_materialization(): - bytewax_config = IntegrationTestRepoConfig( - provider="aws", - online_store={"type": "dynamodb", "region": "us-west-2"}, - offline_store_creator=RedshiftDataSourceCreator, - batch_engine={ - "type": "bytewax", - }, - registry_location=RegistryLocation.S3, - ) - bytewax_environment = construct_test_environment(bytewax_config, None) - - df = create_basic_driver_dataset() - ds = bytewax_environment.data_source_creator.create_data_source( - df, - bytewax_environment.feature_store.project, - field_mapping={"ts_1": "ts"}, - ) - - fs = bytewax_environment.feature_store - driver = Entity( - name="driver_id", - join_key="driver_id", - value_type=ValueType.INT64, - ) - - driver_stats_fv = FeatureView( - name="driver_hourly_stats", - entities=["driver_id"], - ttl=timedelta(weeks=52), - features=[Feature(name="value", dtype=ValueType.FLOAT)], - batch_source=ds, - ) - - try: - fs.apply([driver, driver_stats_fv]) - - # materialization is run in two steps and - # we use timestamp from generated dataframe as a split point - split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1) - - print(f"Split datetime: {split_dt}") - - validate_offline_online_store_consistency(fs, driver_stats_fv, split_dt) - finally: - fs.teardown() diff --git a/setup.py b/setup.py index af4d58a2782..610e2a66ca7 100644 --- a/setup.py +++ b/setup.py @@ -90,8 +90,6 @@ AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] -BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] - KUBERNETES_REQUIRED = ["kubernetes<=20.13.0"] SNOWFLAKE_REQUIRED = [ @@ -195,7 +193,6 @@ + GCP_REQUIRED + REDIS_REQUIRED + AWS_REQUIRED - + BYTEWAX_REQUIRED + KUBERNETES_REQUIRED + SNOWFLAKE_REQUIRED + SPARK_REQUIRED @@ -358,7 +355,6 @@ def run(self): "ci": CI_REQUIRED, "gcp": GCP_REQUIRED, "aws": AWS_REQUIRED, - "bytewax": BYTEWAX_REQUIRED, "k8s": KUBERNETES_REQUIRED, "redis": REDIS_REQUIRED, "snowflake": SNOWFLAKE_REQUIRED, From a683939a26e61850a6e1ab14a5e563e9b848019b Mon Sep 17 00:00:00 2001 From: Patrick McGleenon Date: Wed, 17 Apr 2024 07:16:19 +0100 Subject: [PATCH 05/16] docs: Updated hazelcast docs (#4049) --- docs/reference/online-stores/hazelcast.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/reference/online-stores/hazelcast.md b/docs/reference/online-stores/hazelcast.md index e7a6cf757c4..c2fb2d898a8 100644 --- a/docs/reference/online-stores/hazelcast.md +++ b/docs/reference/online-stores/hazelcast.md @@ -2,10 +2,8 @@ ## Description -The Hazelcast online store is in alpha development. - The [Hazelcast](htpps://hazelcast.com) online store provides support for materializing feature values into a Hazelcast cluster for serving online features in real-time. -In order to use Hazelcast as an online store, you need to have a running Hazelcast cluster. You can create a cluster using Hazelcast Viridian Serverless. See this [getting started](https://hazelcast.com/get-started/) page for more details. +In order to use Hazelcast as an online store, you need to have a running Hazelcast cluster. See this [getting started](https://hazelcast.com/get-started/) page for more details. * Each feature view is mapped one-to-one to a specific Hazelcast IMap * This implementation inherits all strengths of Hazelcast such as high availability, fault-tolerance, and data distribution. From b08b8d5ce226cceae5e874a287db300f6fb9d41b Mon Sep 17 00:00:00 2001 From: nanohanno <44575187+nanohanno@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:47:42 +0200 Subject: [PATCH 06/16] feat: Add tags to dynamodb config (#4100) * Add tags to dynamodb config Signed-off-by: hkuepers * Only add tags to dynamodb when configured Signed-off-by: hkuepers --------- Signed-off-by: hkuepers Co-authored-by: hkuepers --- .../feast/infra/online_stores/dynamodb.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index a049189de7f..4a74f9e27f3 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -65,6 +65,9 @@ class DynamoDBOnlineStoreConfig(FeastConfigBaseModel): consistent_reads: StrictBool = False """Whether to read from Dynamodb by forcing consistent reads""" + tags: Union[Dict[str, str], None] = None + """AWS resource tags added to each table""" + class DynamoDBOnlineStore(OnlineStore): """ @@ -104,7 +107,18 @@ def update( dynamodb_resource = self._get_dynamodb_resource( online_config.region, online_config.endpoint_url ) - + # Add Tags attribute to creation request only if configured to prevent + # TagResource permission issues, even with an empty Tags array. + kwargs = ( + { + "Tags": [ + {"Key": key, "Value": value} + for key, value in online_config.tags.items() + ] + } + if online_config.tags + else {} + ) for table_instance in tables_to_keep: try: dynamodb_resource.create_table( @@ -114,6 +128,7 @@ def update( {"AttributeName": "entity_id", "AttributeType": "S"} ], BillingMode="PAY_PER_REQUEST", + **kwargs, ) except ClientError as ce: # If the table creation fails with ResourceInUseException, From 160e299137201e52fcb06a4d32c2042c32ed74fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:22:33 +0000 Subject: [PATCH 07/16] chore: Bump follow-redirects from 1.15.4 to 1.15.6 in /sdk/python/feast/ui (#4016) chore: Bump follow-redirects in /sdk/python/feast/ui Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.4 to 1.15.6. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.4...v1.15.6) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 0db6b68e244..fe2260742dd 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -5490,9 +5490,9 @@ focus-lock@^0.11.2: tslib "^2.0.3" follow-redirects@^1.0.0: - version "1.15.4" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" - integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.2" From 150f709ac8198b76f1930d6e5ba4d16c9ba1574d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:27:41 +0000 Subject: [PATCH 08/16] chore: Bump follow-redirects from 1.15.4 to 1.15.6 in /ui (#4017) Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.4 to 1.15.6. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.4...v1.15.6) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 5c9e5c17ace..74e842791cd 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -5696,9 +5696,9 @@ focus-lock@^0.11.2: tslib "^2.0.3" follow-redirects@^1.0.0: - version "1.15.4" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" - integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.0" From 9f3c4c453eee22c5a0500bea4a217d1100b9fec1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:10:30 +0000 Subject: [PATCH 09/16] chore: Bump express from 4.18.2 to 4.19.2 in /ui (#4048) Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.18.2...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 74e842791cd..5695441b5e3 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -3553,13 +3553,13 @@ bluebird@^3.5.5, bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -body-parser@1.20.1: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== +body-parser@1.20.2: + version "1.20.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" - content-type "~1.0.4" + content-type "~1.0.5" debug "2.6.9" depd "2.0.0" destroy "1.2.0" @@ -3567,7 +3567,7 @@ body-parser@1.20.1: iconv-lite "0.4.24" on-finished "2.4.1" qs "6.11.0" - raw-body "2.5.1" + raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -4070,6 +4070,11 @@ content-type@~1.0.4: resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== +content-type@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" + integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== + convert-source-map@^1.4.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0: version "1.8.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.8.0.tgz#f3373c32d21b4d780dd8004514684fb791ca4369" @@ -4082,10 +4087,10 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== cookie@^0.4.1: version "0.4.2" @@ -5466,16 +5471,16 @@ expect@^27.4.6: jest-message-util "^27.4.6" express@^4.17.1: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.1" + body-parser "1.20.2" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.5.0" + cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" @@ -9127,10 +9132,10 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== dependencies: bytes "3.1.2" http-errors "2.0.0" From 1a0fe9e8584bf509f20d4da33637f1908b1df6b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:13:53 -0500 Subject: [PATCH 10/16] chore: Bump webpack-dev-middleware from 5.3.0 to 5.3.4 in /ui (#4033) Bumps [webpack-dev-middleware](https://github.com/webpack/webpack-dev-middleware) from 5.3.0 to 5.3.4. - [Release notes](https://github.com/webpack/webpack-dev-middleware/releases) - [Changelog](https://github.com/webpack/webpack-dev-middleware/blob/v5.3.4/CHANGELOG.md) - [Commits](https://github.com/webpack/webpack-dev-middleware/compare/v5.3.0...v5.3.4) --- updated-dependencies: - dependency-name: webpack-dev-middleware dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 5695441b5e3..60a98cd347b 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -5786,6 +5786,11 @@ fs-monkey@1.0.3: resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.3.tgz#ae3ac92d53bb328efe0e9a1d9541f6ad8d48e2d3" integrity sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q== +fs-monkey@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.5.tgz#fe450175f0db0d7ea758102e1d84096acb925788" + integrity sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew== + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -7709,13 +7714,20 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -memfs@^3.1.2, memfs@^3.2.2: +memfs@^3.1.2: version "3.4.1" resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.4.1.tgz#b78092f466a0dce054d63d39275b24c71d3f1305" integrity sha512-1c9VPVvW5P7I85c35zAdEr1TD5+F11IToIHIlrVIcflfnzPkJa0ZoYEoEdYDP8KgPFoSZ/opDrUsAoZWym3mtw== dependencies: fs-monkey "1.0.3" +memfs@^3.4.3: + version "3.6.0" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.6.0.tgz#d7a2110f86f79dd950a8b6df6d57bc984aa185f6" + integrity sha512-EGowvkkgbMcIChjMTMkESFDbZeSh8xZ7kNSF0hAiAN4Jh6jgHCRS0Ga/+C8y6Au+oqpezRHCfPsmJ2+DwAgiwQ== + dependencies: + fs-monkey "^1.0.4" + "memoize-one@>=3.1.1 <6", memoize-one@^5.1.1: version "5.2.1" resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" @@ -11241,12 +11253,12 @@ webidl-conversions@^6.1.0: integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== webpack-dev-middleware@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.0.tgz#8fc02dba6e72e1d373eca361623d84610f27be7c" - integrity sha512-MouJz+rXAm9B1OTOYaJnn6rtD/lWZPy2ufQCH3BPs8Rloh/Du6Jze4p7AeLYHkVi0giJnYLaSGDC7S+GM9arhg== + version "5.3.4" + resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" + integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== dependencies: colorette "^2.0.10" - memfs "^3.2.2" + memfs "^3.4.3" mime-types "^2.1.31" range-parser "^1.2.1" schema-utils "^4.0.0" From fdc2fae9fe9dc0013d18bd2e0b4c6b82e53e9290 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:14:13 -0500 Subject: [PATCH 11/16] chore: Bump webpack-dev-middleware from 5.3.1 to 5.3.4 in /sdk/python/feast/ui (#4034) chore: Bump webpack-dev-middleware in /sdk/python/feast/ui Bumps [webpack-dev-middleware](https://github.com/webpack/webpack-dev-middleware) from 5.3.1 to 5.3.4. - [Release notes](https://github.com/webpack/webpack-dev-middleware/releases) - [Changelog](https://github.com/webpack/webpack-dev-middleware/blob/v5.3.4/CHANGELOG.md) - [Commits](https://github.com/webpack/webpack-dev-middleware/compare/v5.3.1...v5.3.4) --- updated-dependencies: - dependency-name: webpack-dev-middleware dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/yarn.lock | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index fe2260742dd..bc00e87a980 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -5566,6 +5566,11 @@ fs-monkey@1.0.3: resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.3.tgz#ae3ac92d53bb328efe0e9a1d9541f6ad8d48e2d3" integrity sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q== +fs-monkey@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.5.tgz#fe450175f0db0d7ea758102e1d84096acb925788" + integrity sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew== + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -7327,13 +7332,20 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= -memfs@^3.1.2, memfs@^3.4.1: +memfs@^3.1.2: version "3.4.1" resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.4.1.tgz#b78092f466a0dce054d63d39275b24c71d3f1305" integrity sha512-1c9VPVvW5P7I85c35zAdEr1TD5+F11IToIHIlrVIcflfnzPkJa0ZoYEoEdYDP8KgPFoSZ/opDrUsAoZWym3mtw== dependencies: fs-monkey "1.0.3" +memfs@^3.4.3: + version "3.6.0" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.6.0.tgz#d7a2110f86f79dd950a8b6df6d57bc984aa185f6" + integrity sha512-EGowvkkgbMcIChjMTMkESFDbZeSh8xZ7kNSF0hAiAN4Jh6jgHCRS0Ga/+C8y6Au+oqpezRHCfPsmJ2+DwAgiwQ== + dependencies: + fs-monkey "^1.0.4" + "memoize-one@>=3.1.1 <6", memoize-one@^5.1.1: version "5.2.1" resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" @@ -10589,12 +10601,12 @@ webidl-conversions@^6.1.0: integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== webpack-dev-middleware@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.1.tgz#aa079a8dedd7e58bfeab358a9af7dab304cee57f" - integrity sha512-81EujCKkyles2wphtdrnPg/QqegC/AtqNH//mQkBYSMqwFVCQrxM6ktB2O/SPlZy7LqeEfTbV3cZARGQz6umhg== + version "5.3.4" + resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" + integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== dependencies: colorette "^2.0.10" - memfs "^3.4.1" + memfs "^3.4.3" mime-types "^2.1.31" range-parser "^1.2.1" schema-utils "^4.0.0" From 17272d15089f32341e1caa0833b3cb9d3347d97b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:33:06 +0000 Subject: [PATCH 12/16] chore: Bump protobufjs from 7.2.4 to 7.2.5 in /ui (#4094) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.2.4 to 7.2.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.2.4...protobufjs-v7.2.5) --- updated-dependencies: - dependency-name: protobufjs dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 60a98cd347b..0bfdb63184e 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -9048,9 +9048,9 @@ protobufjs-cli@^1.0.2: uglify-js "^3.7.7" protobufjs@^7.1.1: - version "7.2.4" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.4.tgz#3fc1ec0cdc89dd91aef9ba6037ba07408485c3ae" - integrity sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ== + version "7.2.5" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.5.tgz#45d5c57387a6d29a17aab6846dcc283f9b8e7f2d" + integrity sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" From d910033a448c75200d29c01ca6792e75f84d273b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:33:43 +0000 Subject: [PATCH 13/16] chore: Bump protobufjs from 7.2.4 to 7.2.6 in /sdk/python/feast/ui (#4093) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.2.4 to 7.2.6. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.2.4...protobufjs-v7.2.6) --- updated-dependencies: - dependency-name: protobufjs dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index bc00e87a980..06677dc1aea 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -8568,9 +8568,9 @@ property-information@^5.0.0, property-information@^5.3.0: xtend "^4.0.0" protobufjs@^7.1.1: - version "7.2.4" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.4.tgz#3fc1ec0cdc89dd91aef9ba6037ba07408485c3ae" - integrity sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ== + version "7.2.6" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.6.tgz#4a0ccd79eb292717aacf07530a07e0ed20278215" + integrity sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" From 44dc82acf3beb25bc0bf077a9ed6bb2614152515 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:42:52 +0000 Subject: [PATCH 14/16] chore: Bump gunicorn from 21.2.0 to 22.0.0 in /sdk/python/requirements (#4106) Bumps [gunicorn](https://github.com/benoitc/gunicorn) from 21.2.0 to 22.0.0. - [Release notes](https://github.com/benoitc/gunicorn/releases) - [Commits](https://github.com/benoitc/gunicorn/compare/21.2.0...22.0.0) --- updated-dependencies: - dependency-name: gunicorn dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index ab6a670f32b..424a433be46 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -289,7 +289,7 @@ grpcio-testing==1.62.1 # via feast (setup.py) grpcio-tools==1.62.1 # via feast (setup.py) -gunicorn==21.2.0 ; platform_system != "Windows" +gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 1ac3aca8dbe..ba3574ede8b 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -52,7 +52,7 @@ fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 ; platform_system != "Windows" +gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 9178b1de127..88a67595835 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -289,7 +289,7 @@ grpcio-testing==1.62.1 # via feast (setup.py) grpcio-tools==1.62.1 # via feast (setup.py) -gunicorn==21.2.0 ; platform_system != "Windows" +gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index ebf00789aec..a003937823e 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -52,7 +52,7 @@ fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 ; platform_system != "Windows" +gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via uvicorn From f3ad2888749be4008c4925343afa12c224bde5d0 Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Wed, 17 Apr 2024 17:21:24 +0200 Subject: [PATCH 15/16] docs: Fix command to run python tests on documentation (#4111) docs: Update command to run python tests on documentation Signed-off-by: Breno Costa --- .../customizing-feast/adding-a-new-offline-store.md | 2 +- .../customizing-feast/adding-support-for-a-new-online-store.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md index b2818b748f8..9c4ed1c45a3 100644 --- a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md +++ b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md @@ -381,7 +381,7 @@ Even if you have created the `OfflineStore` class in a separate repo, you can st 2. Make sure that your offline store doesn't break any unit tests first by running: ``` - make test-python + make test-python-unit ``` 3. Next, set up your offline store to run the universal integration tests. These are integration tests specifically intended to test offline and online stores against Feast API functionality, to ensure that the Feast APIs works with your offline store. diff --git a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md index ab88ebaa203..e3f1fcf4286 100644 --- a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md @@ -319,7 +319,7 @@ Even if you have created the `OnlineStore` class in a separate repo, you can sti 1. In the Feast submodule, we can run all the unit tests and make sure they pass: ``` - make test-python + make test-python-unit ``` 2. The universal tests, which are integration tests specifically intended to test offline and online stores, should be run against Feast to ensure that the Feast APIs works with your online store. * Feast parametrizes integration tests using the `FULL_REPO_CONFIGS` variable defined in `sdk/python/tests/integration/feature_repos/repo_configuration.py` which stores different online store classes for testing. From 0fb23514c052b415e255ebc9a7f1affc6f6d0250 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Wed, 17 Apr 2024 16:04:38 +0000 Subject: [PATCH 16/16] chore(release): release 0.37.0 # [0.37.0](https://github.com/feast-dev/feast/compare/v0.36.0...v0.37.0) (2024-04-17) ### Bug Fixes * Pgvector patch ([#4103](https://github.com/feast-dev/feast/issues/4103)) ([5c4a9c5](https://github.com/feast-dev/feast/commit/5c4a9c57fa42ee5688fb6b428cedb416a7dbf185)) * Remove top-level grpc import in cli ([#4107](https://github.com/feast-dev/feast/issues/4107)) ([4362b6c](https://github.com/feast-dev/feast/commit/4362b6cc857ceafe60d58a14f2dfe006a83effb8)) ### Features * Add tags to dynamodb config ([#4100](https://github.com/feast-dev/feast/issues/4100)) ([b08b8d5](https://github.com/feast-dev/feast/commit/b08b8d5ce226cceae5e874a287db300f6fb9d41b)) --- CHANGELOG.md | 13 +++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 ++-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- .../charts/feast/charts/feature-server/values.yaml | 2 +- .../feast/charts/transformation-service/Chart.yaml | 4 ++-- .../feast/charts/transformation-service/README.md | 4 ++-- .../feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 17 files changed, 40 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f09ec9e2b7a..6c5fc0f01b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +# [0.37.0](https://github.com/feast-dev/feast/compare/v0.36.0...v0.37.0) (2024-04-17) + + +### Bug Fixes + +* Pgvector patch ([#4103](https://github.com/feast-dev/feast/issues/4103)) ([5c4a9c5](https://github.com/feast-dev/feast/commit/5c4a9c57fa42ee5688fb6b428cedb416a7dbf185)) +* Remove top-level grpc import in cli ([#4107](https://github.com/feast-dev/feast/issues/4107)) ([4362b6c](https://github.com/feast-dev/feast/commit/4362b6cc857ceafe60d58a14f2dfe006a83effb8)) + + +### Features + +* Add tags to dynamodb config ([#4100](https://github.com/feast-dev/feast/issues/4100)) ([b08b8d5](https://github.com/feast-dev/feast/commit/b08b8d5ce226cceae5e874a287db300f6fb9d41b)) + # [0.36.0](https://github.com/feast-dev/feast/compare/v0.35.0...v0.36.0) (2024-04-16) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index d7a7e306519..bd4bc606a70 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.36.0 +version: 0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 48018b88d58..fa6d89361c0 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.36.0` +Current chart version is `0.37.0` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.36.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.37.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 6177e746fb4..0de071ef3db 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.36.0 + tag: 0.37.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 6e146a442b9..26a00d80c63 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.36.0 +version: 0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 241bafdc65d..47959047dc2 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.36.0` +Feature store for machine learning Current chart version is `0.37.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.36.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.36.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 14da5829338..f2f8c748dc4 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.36.0 -appVersion: v0.36.0 +version: 0.37.0 +appVersion: v0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 8fd847e71ba..531adce92da 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.36.0](https://img.shields.io/badge/Version-0.36.0-informational?style=flat-square) ![AppVersion: v0.36.0](https://img.shields.io/badge/AppVersion-v0.36.0-informational?style=flat-square) +![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.36.0"` | Image tag | +| image.tag | string | `"0.37.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 76bfe2c0cbb..1bf1a03f4a7 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.36.0 + tag: 0.37.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 218726b7f19..056e00473fb 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.36.0 -appVersion: v0.36.0 +version: 0.37.0 +appVersion: v0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 5d4772572d0..4b11861d539 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.36.0](https://img.shields.io/badge/Version-0.36.0-informational?style=flat-square) ![AppVersion: v0.36.0](https://img.shields.io/badge/AppVersion-v0.36.0-informational?style=flat-square) +![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.36.0"` | Image tag | +| image.tag | string | `"0.37.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 8e9fe2f87a4..a04dfeb3e04 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.36.0 + tag: 0.37.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 601d9a7a84b..9a2d3e0e807 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.36.0 + version: 0.37.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.36.0 + version: 0.37.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index b8d82c8e04b..2d7e2c3e7d2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.36.0 + 0.37.0 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 5ba6777007e..cdad51e0608 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.36.0", + "@feast-dev/feast-ui": "0.37.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 06677dc1aea..07472d7bbeb 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.36.0": - version "0.36.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.36.0.tgz#b8cdc5f3e959fa2c787e2f1839aaaec99c739d65" - integrity sha512-R+F1cQI89Nrw64GnePENYMRLcSb6DH24zs5aO57ovGbFQquR0vqJ6UfqjhLlIQpkEE6quV2Zr3QOwVB1GBVL1g== +"@feast-dev/feast-ui@0.37.0": + version "0.37.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.0.tgz#53e04a73835617f23528bd95228190cf022338b0" + integrity sha512-bpW4K6y+XE9np+/zXsYffQWrND7Bx3z1xBj4SSddgX3QkGJyaJeOM9Rfu4ZhkjwNGvfqKaxGz9Ucnhpjv9lrfw== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index babf01b8fa5..9209d7b03c7 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.36.0", + "version": "0.37.0", "private": false, "files": [ "dist"