Skip to content

fix: Adding protobuf<5 as a required dependency due to snowflake limitations #4537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tokoko merged 4 commits into from
Sep 19, 2024
Merged

fix: Adding protobuf<5 as a required dependency due to snowflake limitations #4537

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aa84536
fix: Removed protobuf as a required dependency
Sep 18, 2024
8fae3c3
fix: Removed install-protoc-dependencies target
Sep 18, 2024
048c17c
fix: Ran lock python dependencies to correct dependencies
Sep 18, 2024
b764460
fix: Adding protobuf<5 as a required dependency due to snowflake limi…
Sep 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
fix: Adding protobuf<5 as a required dependency due to snowflake limi… 
…tations

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>
  • Loading branch information
Bhargav Dodla committed Sep 19, 2024
commit b764460af4dfaef1a2b51e8609faba96719dc57e
1 change: 1 addition & 0 deletions .github/workflows/build_wheels.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ jobs:
# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
run: |
pip install -U pip setuptools wheel twine
make install-protoc-dependencies
make build-ui
git status
git restore go.mod go.sum
Expand Down
19 changes: 12 additions & 7 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,16 @@ install-python-ci-dependencies-uv-venv:
uv pip install --no-deps -e .
python setup.py build_python_protos --inplace

install-protoc-dependencies:
pip install "protobuf<5" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"

lock-python-ci-dependencies:
uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt

package-protos:
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos

compile-protos-python:
compile-protos-python: install-protoc-dependencies
python setup.py build_python_protos --inplace

install-python:
Expand All @@ -69,12 +72,14 @@ lock-python-dependencies:
uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt

lock-python-dependencies-all:
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt"
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"
# Remove all existing requirements because we noticed the lock file is not always updated correctly. Removing and running the command again ensures that the lock file is always up to date.
rm -r sdk/python/requirements/*
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt"
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"

benchmark-python:
IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests
Expand Down
17 changes: 9 additions & 8 deletions sdk/python/requirements/py3.10-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt
# uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt
aiobotocore==2.15.0
# via feast (setup.py)
aiohappyeyeballs==2.4.0
Expand All @@ -16,7 +16,7 @@ altair==4.2.2
# via great-expectations
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# httpx
# jupyter-server
Expand Down Expand Up @@ -271,15 +271,15 @@ grpcio==1.66.1
# grpcio-status
# grpcio-testing
# grpcio-tools
grpcio-health-checking==1.66.1
grpcio-health-checking==1.62.3
# via feast (setup.py)
grpcio-reflection==1.66.1
grpcio-reflection==1.62.3
# via feast (setup.py)
grpcio-status==1.66.1
grpcio-status==1.62.3
# via google-api-core
grpcio-testing==1.66.1
grpcio-testing==1.62.3
# via feast (setup.py)
grpcio-tools==1.66.1
grpcio-tools==1.62.3
# via feast (setup.py)
gunicorn==23.0.0
# via feast (setup.py)
Expand Down Expand Up @@ -573,8 +573,9 @@ proto-plus==1.24.0
# google-cloud-bigquery-storage
# google-cloud-bigtable
# google-cloud-datastore
protobuf==5.28.1
protobuf==4.25.5
# via
# feast (setup.py)
# google-api-core
# google-cloud-bigquery-storage
# google-cloud-bigtable
Expand Down
12 changes: 4 additions & 8 deletions sdk/python/requirements/py3.10-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt
# uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# starlette
# watchfiles
Expand Down Expand Up @@ -67,8 +67,6 @@ mypy==1.11.2
# via sqlalchemy
mypy-extensions==1.0.0
# via mypy
mypy-protobuf==3.6.0
# via feast (setup.py)
numpy==1.26.4
# via
# feast (setup.py)
Expand All @@ -88,8 +86,8 @@ partd==1.4.2
# via dask
prometheus-client==0.20.0
# via feast (setup.py)
protobuf==5.28.1
# via mypy-protobuf
protobuf==4.25.5
# via feast (setup.py)
psutil==6.0.0
# via feast (setup.py)
pyarrow==17.0.0
Expand Down Expand Up @@ -151,8 +149,6 @@ tqdm==4.66.5
# via feast (setup.py)
typeguard==4.3.0
# via feast (setup.py)
types-protobuf==5.27.0.20240907
# via mypy-protobuf
typing-extensions==4.12.2
# via
# anyio
Expand Down
17 changes: 9 additions & 8 deletions sdk/python/requirements/py3.11-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt
# uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt
aiobotocore==2.15.0
# via feast (setup.py)
aiohappyeyeballs==2.4.0
Expand All @@ -16,7 +16,7 @@ altair==4.2.2
# via great-expectations
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# httpx
# jupyter-server
Expand Down Expand Up @@ -264,15 +264,15 @@ grpcio==1.66.1
# grpcio-status
# grpcio-testing
# grpcio-tools
grpcio-health-checking==1.66.1
grpcio-health-checking==1.62.3
# via feast (setup.py)
grpcio-reflection==1.66.1
grpcio-reflection==1.62.3
# via feast (setup.py)
grpcio-status==1.66.1
grpcio-status==1.62.3
# via google-api-core
grpcio-testing==1.66.1
grpcio-testing==1.62.3
# via feast (setup.py)
grpcio-tools==1.66.1
grpcio-tools==1.62.3
# via feast (setup.py)
gunicorn==23.0.0
# via feast (setup.py)
Expand Down Expand Up @@ -564,8 +564,9 @@ proto-plus==1.24.0
# google-cloud-bigquery-storage
# google-cloud-bigtable
# google-cloud-datastore
protobuf==5.28.1
protobuf==4.25.5
# via
# feast (setup.py)
# google-api-core
# google-cloud-bigquery-storage
# google-cloud-bigtable
Expand Down
12 changes: 4 additions & 8 deletions sdk/python/requirements/py3.11-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt
# uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# starlette
# watchfiles
Expand Down Expand Up @@ -65,8 +65,6 @@ mypy==1.11.2
# via sqlalchemy
mypy-extensions==1.0.0
# via mypy
mypy-protobuf==3.6.0
# via feast (setup.py)
numpy==1.26.4
# via
# feast (setup.py)
Expand All @@ -86,8 +84,8 @@ partd==1.4.2
# via dask
prometheus-client==0.20.0
# via feast (setup.py)
protobuf==5.28.1
# via mypy-protobuf
protobuf==4.25.5
# via feast (setup.py)
psutil==6.0.0
# via feast (setup.py)
pyarrow==17.0.0
Expand Down Expand Up @@ -147,8 +145,6 @@ tqdm==4.66.5
# via feast (setup.py)
typeguard==4.3.0
# via feast (setup.py)
types-protobuf==5.27.0.20240907
# via mypy-protobuf
typing-extensions==4.12.2
# via
# fastapi
Expand Down
17 changes: 9 additions & 8 deletions sdk/python/requirements/py3.9-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt
# uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt
aiobotocore==2.15.0
# via feast (setup.py)
aiohappyeyeballs==2.4.0
Expand All @@ -16,7 +16,7 @@ altair==4.2.2
# via great-expectations
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# httpx
# jupyter-server
Expand Down Expand Up @@ -273,15 +273,15 @@ grpcio==1.66.1
# grpcio-status
# grpcio-testing
# grpcio-tools
grpcio-health-checking==1.66.1
grpcio-health-checking==1.62.3
# via feast (setup.py)
grpcio-reflection==1.66.1
grpcio-reflection==1.62.3
# via feast (setup.py)
grpcio-status==1.66.1
grpcio-status==1.62.3
# via google-api-core
grpcio-testing==1.66.1
grpcio-testing==1.62.3
# via feast (setup.py)
grpcio-tools==1.66.1
grpcio-tools==1.62.3
# via feast (setup.py)
gunicorn==23.0.0
# via feast (setup.py)
Expand Down Expand Up @@ -581,8 +581,9 @@ proto-plus==1.24.0
# google-cloud-bigquery-storage
# google-cloud-bigtable
# google-cloud-datastore
protobuf==5.28.1
protobuf==4.25.5
# via
# feast (setup.py)
# google-api-core
# google-cloud-bigquery-storage
# google-cloud-bigtable
Expand Down
12 changes: 4 additions & 8 deletions sdk/python/requirements/py3.9-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt
# uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==4.5.0
# via
# starlette
# watchfiles
Expand Down Expand Up @@ -69,8 +69,6 @@ mypy==1.11.2
# via sqlalchemy
mypy-extensions==1.0.0
# via mypy
mypy-protobuf==3.6.0
# via feast (setup.py)
numpy==1.26.4
# via
# feast (setup.py)
Expand All @@ -90,8 +88,8 @@ partd==1.4.2
# via dask
prometheus-client==0.20.0
# via feast (setup.py)
protobuf==5.28.1
# via mypy-protobuf
protobuf==4.25.5
# via feast (setup.py)
psutil==6.0.0
# via feast (setup.py)
pyarrow==17.0.0
Expand Down Expand Up @@ -153,8 +151,6 @@ tqdm==4.66.5
# via feast (setup.py)
typeguard==4.3.0
# via feast (setup.py)
types-protobuf==5.27.0.20240907
# via mypy-protobuf
typing-extensions==4.12.2
# via
# anyio
Expand Down
7 changes: 6 additions & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"click>=7.0.0,<9.0.0",
"colorama>=0.3.9,<1",
"dill~=0.3.0",
"mypy-protobuf>=3.1",
"protobuf<5",
"Jinja2>=2,<4",
"jsonschema",
"mmh3",
Expand Down Expand Up @@ -157,6 +157,8 @@
"virtualenv==20.23.0",
"cryptography>=35.0,<43",
"ruff>=0.3.3",
"protobuf<5",
"mypy-protobuf>=3.1",
"grpcio-tools>=1.56.2,<2",
"grpcio-testing>=1.56.2,<2",
# FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656).
Expand Down Expand Up @@ -400,6 +402,9 @@ def run(self):
entry_points={"console_scripts": ["feast=feast.cli:cli"]},
use_scm_version=use_scm_version,
setup_requires=[
# snowflake udf packages refer to conda packages, not pypi libraries. Conda stack is still on protobuf 4
# So we are adding protobuf<5 as a requirement
"protobuf<5",
"grpcio-tools>=1.56.2,<2",
"mypy-protobuf>=3.1",
"pybindgen==0.22.0",
Expand Down