From a45e10a2558129fcb906fe3e7a03006aeea49d97 Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Thu, 30 Jun 2022 02:11:16 +0200 Subject: [PATCH 01/26] fix: Fixing broken links to feast documentation on java readme and contribution (#2892) Signed-off-by: Breno Costa --- java/CONTRIBUTING.md | 4 ++-- java/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/CONTRIBUTING.md b/java/CONTRIBUTING.md index 86eacfef419..f6c789d9847 100644 --- a/java/CONTRIBUTING.md +++ b/java/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Development Guide: feast-java -> The higher level [Development Guide](https://docs.feast.dev/contributing/development-guide) +> The higher level [Development Guide](https://docs.feast.dev/v/master/project/development-guide) > gives contributing to Feast codebase as a whole. ### Overview @@ -9,7 +9,7 @@ the feast-java Repository: - [Feast Java Client](#feast-java-client) > Don't see the Feast component that you want to contribute to here? -> Check out the [Development Guide](https://docs.feast.dev/contributing/development-guide) +> Check out the [Development Guide](https://docs.feast.dev/v/master/project/development-guide) > to learn how Feast components are distributed over multiple repositories. #### Common Setup diff --git a/java/README.md b/java/README.md index ff5a1b85539..8c3d93628e3 100644 --- a/java/README.md +++ b/java/README.md @@ -13,8 +13,8 @@ The process of ingesting data into the online store (Redis) is decoupled from th ### Contributing Guides on Contributing: -- [Contribution Process for Feast](https://docs.feast.dev/v/master/contributing/contributing) -- [Development Guide for Feast](https://docs.feast.dev/contributing/development-guide) +- [Contribution Process for Feast](https://docs.feast.dev/v/master/project/contributing) +- [Development Guide for Feast](https://docs.feast.dev/v/master/project/development-guide) - [Development Guide for feast-java (this repository)](CONTRIBUTING.md) ### Installing using Helm From b9190b9f37a9a05421c3a288e9ba1807124dcb12 Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Thu, 30 Jun 2022 03:40:15 +0200 Subject: [PATCH 02/26] fix: Change numpy version on setup.py and upgrade it to resolve dependabot warning (#2887) * Upgrade numpy version on setup.py and resolve dependabot warning Signed-off-by: Breno Costa * Upgrade python base image from 3.7 to 3.8 Signed-off-by: Breno Costa --- .../src/test/resources/docker-compose/feast10/Dockerfile | 2 +- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.8-ci-requirements.txt | 2 +- sdk/python/requirements/py3.8-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- setup.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index dc26c804a9a..dee7dcf84c4 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.8 WORKDIR /usr/src/ diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 3bdf468bb26..bcd1095d073 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -384,7 +384,7 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.6.0 # via pre-commit -numpy==1.21.6 +numpy==1.22.0 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 15ee46aff53..dc60ea1e70e 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -89,7 +89,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.21.6 +numpy==1.22.0 # via # feast (setup.py) # pandas diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index dd21fae0a2d..2be1e80464b 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -390,7 +390,7 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.6.0 # via pre-commit -numpy==1.21.6 +numpy==1.22.0 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 7756acad31e..8ba9f4c35de 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -91,7 +91,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.21.6 +numpy==1.22.0 # via # feast (setup.py) # pandas diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index f9f65633f00..e0fbb8092dd 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -384,7 +384,7 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.6.0 # via pre-commit -numpy==1.21.6 +numpy==1.22.0 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index f5c15dad5d4..da1a7f4345c 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -89,7 +89,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.21.6 +numpy==1.22.0 # via # feast (setup.py) # pandas diff --git a/setup.py b/setup.py index d32b13deed0..55a715b53af 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ "Jinja2>=2,<4", "jsonschema", "mmh3", - "numpy<1.22,<2", + "numpy>=1.22,<2", "pandas>=1,<2", "pandavro==1.5.*", "protobuf>=3.10,<3.20", From a5f68bf1019daa9b04a59a80012aea68e277eb93 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 29 Jun 2022 20:33:16 -0700 Subject: [PATCH 03/26] chore: Verification workflow in build wheels that ensures the wheels built are not development and the correct versions (#2893) * Test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix workflow Signed-off-by: Kevin Zhang * Fix workflow Signed-off-by: Kevin Zhang * Test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 6ee866781dc..b578501b596 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -1,7 +1,13 @@ name: build_wheels # Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml" -on: [workflow_dispatch, workflow_call] +# Developers who are starting a new release should use this workflow to ensure wheels will be built correctly. +# Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly. +on: + workflow_dispatch: + tags: + - 'v*.*.*' + workflow_call: jobs: get-version: @@ -134,7 +140,7 @@ jobs: verify-python-wheels: runs-on: ${{ matrix.os }} - needs: [build-python-wheel, build-source-distribution] + needs: [build-python-wheel, build-source-distribution, get-version] strategy: matrix: os: [ubuntu-latest, macos-10.15 ] @@ -153,6 +159,7 @@ jobs: else echo "Succeeded!" fi + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - name: Setup Python id: setup-python @@ -190,6 +197,19 @@ jobs: - name: Install OS X dependencies if: matrix.os == 'macos-10.15' run: brew install coreutils + # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of. + - name: Validate Feast Version + run: | + VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+' + OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$' + VERSION_OUTPUT=$(feast version) + VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX") + OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX") + if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then + echo "Correct Feast Version Installed" + else + echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION." + fi - name: Smoke test run: | feast init test_repo @@ -198,6 +218,7 @@ jobs: echo "$TEST_SCRIPT" > run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui + # We disable this test for the Python 3.10 binary since it does not include Go. - name: Smoke test with go if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest' From f726c967dc132902423d336d990715c3033fe196 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 30 Jun 2022 10:16:30 -0700 Subject: [PATCH 04/26] fix: Fix grpc and update protobuf (#2894) * Test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix workflow Signed-off-by: Kevin Zhang * Fix workflow Signed-off-by: Kevin Zhang * Test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix protobuf pin version and add path variable Signed-off-by: Kevin Zhang * Revert setup Signed-off-by: Kevin Zhang * Update 3.8 Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Update 3.10 Signed-off-by: Kevin Zhang * update Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix test Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Update go with grpc 1.47 Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 4 +- Makefile | 2 +- go.mod | 2 +- go.sum | 5 +- pyproject.toml | 2 +- sdk/python/feast/proto_json.py | 8 +-- .../requirements/py3.10-ci-requirements.txt | 54 +++++++++--------- .../requirements/py3.10-requirements.txt | 18 +++--- .../requirements/py3.8-ci-requirements.txt | 56 +++++++++---------- .../requirements/py3.8-requirements.txt | 20 +++---- .../requirements/py3.9-ci-requirements.txt | 54 +++++++++--------- .../requirements/py3.9-requirements.txt | 18 +++--- sdk/python/tests/unit/test_proto_json.py | 2 +- setup.py | 16 +++--- 14 files changed, 126 insertions(+), 135 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index b578501b596..ea93cc85a44 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -186,7 +186,7 @@ jobs: env: COMPILE_GO: "True" run: | - pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' + pip install 'grpcio-tools==1.47.0' 'pybindgen==0.22.0' go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 pip install dist/*tar.gz @@ -209,6 +209,7 @@ jobs: echo "Correct Feast Version Installed" else echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION." + exit 1 fi - name: Smoke test run: | @@ -218,7 +219,6 @@ jobs: echo "$TEST_SCRIPT" > run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui - # We disable this test for the Python 3.10 binary since it does not include Go. - name: Smoke test with go if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest' diff --git a/Makefile b/Makefile index 88f04aa95d8..176e2cb3545 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ install-go-ci-dependencies: python -m pip install pybindgen==0.22.0 install-protoc-dependencies: - pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0 + pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos diff --git a/go.mod b/go.mod index fbbc95e1bfe..90ddb93e21c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 - google.golang.org/grpc v1.45.0 + google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.0 ) diff --git a/go.sum b/go.sum index 698d1ef640d..933ecf6b29c 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -83,6 +84,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 h1:Go714ObVP1O+a6qK7haXVL28QNm6WMD8bwnN9EA8PlM= @@ -588,8 +590,9 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pyproject.toml b/pyproject.toml index 64394a558a6..8ba72544404 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=60", "wheel", "setuptools_scm>=6.2", "grpcio", "grpcio-tools==1.44.0", "mypy-protobuf==3.1", "sphinx!=4.0.0"] +requires = ["setuptools>=60", "wheel", "setuptools_scm>=6.2", "grpcio", "grpcio-tools==1.47.0", "mypy-protobuf==3.1", "sphinx!=4.0.0"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/sdk/python/feast/proto_json.py b/sdk/python/feast/proto_json.py index 44e004cb036..58b77edf8ba 100644 --- a/sdk/python/feast/proto_json.py +++ b/sdk/python/feast/proto_json.py @@ -70,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return value def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage, + parser: _Parser, value: JsonObject, message: ProtoMessage, path: str ) -> None: if value is None: message.null_val = 0 @@ -142,11 +142,11 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return [printer._MessageToJsonObject(item) for item in message.val] def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage, + parser: _Parser, value: JsonObject, message: ProtoMessage, path: str ) -> None: array = value if isinstance(value, list) else value["val"] for item in array: - parser.ConvertMessage(item, message.val.add()) + parser.ConvertMessage(item, message.val.add(), path) _patch_proto_json_encoding(RepeatedValue, to_json_object, from_json_object) @@ -183,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return list(message.val) def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage, + parser: _Parser, value: JsonObject, message: ProtoMessage, path: str ) -> None: array = value if isinstance(value, list) else value["val"] message.val.extend(array) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index bcd1095d073..e0b05ecaa08 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -90,7 +90,9 @@ botocore==1.23.24 bowler==0.9.0 # via feast (setup.py) build==0.8.0 - # via feast (setup.py) + # via + # feast (setup.py) + # pip-tools cachecontrol==0.12.11 # via firebase-admin cachetools==4.2.4 @@ -173,7 +175,7 @@ executing==0.8.3 # via stack-data fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -209,7 +211,7 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.51.0 +google-api-python-client==2.52.0 # via firebase-admin google-auth==1.35.0 # via @@ -257,8 +259,6 @@ googleapis-common-protos==1.56.3 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -269,9 +269,9 @@ grpcio==1.47.0 # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-testing==1.44.0 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.44.0 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -295,7 +295,7 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx -importlib-metadata==4.11.4 +importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest @@ -322,7 +322,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.0 +jsonschema==4.6.1 # via # altair # feast (setup.py) @@ -342,7 +342,7 @@ mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.2 +mistune==2.0.3 # via great-expectations mmh3==3.0.0 # via feast (setup.py) @@ -350,7 +350,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.14 +moto==3.1.16 # via feast (setup.py) msal==1.18.0 # via @@ -382,7 +382,7 @@ mysqlclient==2.1.1 # via feast (setup.py) nbformat==5.4.0 # via great-expectations -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit numpy==1.22.0 # via @@ -426,14 +426,12 @@ pathspec==0.9.0 pbr==5.9.0 # via mock pep517==0.12.0 - # via - # build - # pip-tools + # via build pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.6.2 +pip-tools==6.7.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -445,7 +443,7 @@ portalocker==2.4.0 # via msal-extensions pre-commit==2.19.0 # via feast (setup.py) -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.30 # via ipython proto-plus==1.20.6 # via @@ -454,7 +452,7 @@ proto-plus==1.20.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -588,7 +586,7 @@ redis==4.2.2 # via feast (setup.py) regex==2022.6.2 # via black -requests==2.28.0 +requests==2.28.1 # via # adal # adlfs @@ -665,7 +663,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -724,17 +722,17 @@ types-protobuf==3.19.22 # mypy-protobuf types-python-dateutil==2.8.18 # via feast (setup.py) -types-pytz==2022.1.0 +types-pytz==2022.1.1 # via feast (setup.py) -types-pyyaml==6.0.8 +types-pyyaml==6.0.9 # via feast (setup.py) -types-redis==4.3.2 +types-redis==4.3.3 # via feast (setup.py) -types-requests==2.27.31 +types-requests==2.28.0 # via feast (setup.py) -types-setuptools==57.4.17 +types-setuptools==57.4.18 # via feast (setup.py) -types-tabulate==0.8.10 +types-tabulate==0.8.11 # via feast (setup.py) types-urllib3==1.26.15 # via types-requests @@ -759,11 +757,11 @@ urllib3==1.26.9 # minio # requests # responses -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.14.1 +virtualenv==20.15.1 # via pre-commit volatile==2.1.0 # via bowler diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index dc60ea1e70e..d4a4425aece 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -22,7 +22,7 @@ cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.0 # via requests click==8.0.1 # via @@ -40,7 +40,7 @@ dill==0.3.5.1 # via feast (setup.py) fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -50,15 +50,13 @@ fsspec==2022.5.0 # via dask google-api-core==2.8.2 # via feast (setup.py) -google-auth==2.8.0 +google-auth==2.9.0 # via google-api-core googleapis-common-protos==1.56.3 # via # feast (setup.py) # google-api-core # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -75,7 +73,7 @@ idna==3.3 # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.0 +jsonschema==4.6.1 # via feast (setup.py) locket==1.0.0 # via partd @@ -107,7 +105,7 @@ partd==1.2.0 # via dask proto-plus==1.20.6 # via feast (setup.py) -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -144,7 +142,7 @@ pyyaml==6.0 # dask # feast (setup.py) # uvicorn -requests==2.28.0 +requests==2.28.1 # via google-api-core rsa==4.8 # via google-auth @@ -156,7 +154,7 @@ six==1.16.0 # python-dateutil sniffio==1.2.0 # via anyio -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -187,7 +185,7 @@ typing-extensions==4.2.0 # sqlalchemy2-stubs urllib3==1.26.9 # via requests -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 2be1e80464b..813420a121f 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -94,7 +94,9 @@ botocore==1.23.24 bowler==0.9.0 # via feast (setup.py) build==0.8.0 - # via feast (setup.py) + # via + # feast (setup.py) + # pip-tools cachecontrol==0.12.11 # via firebase-admin cachetools==4.2.4 @@ -177,7 +179,7 @@ executing==0.8.3 # via stack-data fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -213,7 +215,7 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.51.0 +google-api-python-client==2.52.0 # via firebase-admin google-auth==1.35.0 # via @@ -261,8 +263,6 @@ googleapis-common-protos==1.56.3 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -273,9 +273,9 @@ grpcio==1.47.0 # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-testing==1.44.0 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.44.0 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -299,7 +299,7 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx -importlib-metadata==4.11.4 +importlib-metadata==4.12.0 # via great-expectations importlib-resources==5.8.0 # via jsonschema @@ -328,7 +328,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.0 +jsonschema==4.6.1 # via # altair # feast (setup.py) @@ -348,7 +348,7 @@ mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.2 +mistune==2.0.3 # via great-expectations mmh3==3.0.0 # via feast (setup.py) @@ -356,7 +356,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.14 +moto==3.1.16 # via feast (setup.py) msal==1.18.0 # via @@ -388,9 +388,9 @@ mysqlclient==2.1.1 # via feast (setup.py) nbformat==5.4.0 # via great-expectations -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit -numpy==1.22.0 +numpy==1.23.0 # via # altair # feast (setup.py) @@ -432,14 +432,12 @@ pathspec==0.9.0 pbr==5.9.0 # via mock pep517==0.12.0 - # via - # build - # pip-tools + # via build pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.6.2 +pip-tools==6.7.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -451,7 +449,7 @@ portalocker==2.4.0 # via msal-extensions pre-commit==2.19.0 # via feast (setup.py) -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.30 # via ipython proto-plus==1.20.6 # via @@ -460,7 +458,7 @@ proto-plus==1.20.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -594,7 +592,7 @@ redis==4.2.2 # via feast (setup.py) regex==2022.6.2 # via black -requests==2.28.0 +requests==2.28.1 # via # adal # adlfs @@ -673,7 +671,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -732,17 +730,17 @@ types-protobuf==3.19.22 # mypy-protobuf types-python-dateutil==2.8.18 # via feast (setup.py) -types-pytz==2022.1.0 +types-pytz==2022.1.1 # via feast (setup.py) -types-pyyaml==6.0.8 +types-pyyaml==6.0.9 # via feast (setup.py) -types-redis==4.3.2 +types-redis==4.3.3 # via feast (setup.py) -types-requests==2.27.31 +types-requests==2.28.0 # via feast (setup.py) -types-setuptools==57.4.17 +types-setuptools==57.4.18 # via feast (setup.py) -types-tabulate==0.8.10 +types-tabulate==0.8.11 # via feast (setup.py) types-urllib3==1.26.15 # via types-requests @@ -769,11 +767,11 @@ urllib3==1.26.9 # minio # requests # responses -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.14.1 +virtualenv==20.15.1 # via pre-commit volatile==2.1.0 # via bowler diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 8ba9f4c35de..56e5e3adba4 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -22,7 +22,7 @@ cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.0 # via requests click==8.0.1 # via @@ -40,7 +40,7 @@ dill==0.3.5.1 # via feast (setup.py) fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -50,15 +50,13 @@ fsspec==2022.5.0 # via dask google-api-core==2.8.2 # via feast (setup.py) -google-auth==2.8.0 +google-auth==2.9.0 # via google-api-core googleapis-common-protos==1.56.3 # via # feast (setup.py) # google-api-core # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -77,7 +75,7 @@ importlib-resources==5.8.0 # via jsonschema jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.0 +jsonschema==4.6.1 # via feast (setup.py) locket==1.0.0 # via partd @@ -91,7 +89,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.22.0 +numpy==1.23.0 # via # feast (setup.py) # pandas @@ -109,7 +107,7 @@ partd==1.2.0 # via dask proto-plus==1.20.6 # via feast (setup.py) -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -146,7 +144,7 @@ pyyaml==6.0 # dask # feast (setup.py) # uvicorn -requests==2.28.0 +requests==2.28.1 # via google-api-core rsa==4.8 # via google-auth @@ -158,7 +156,7 @@ six==1.16.0 # python-dateutil sniffio==1.2.0 # via anyio -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -190,7 +188,7 @@ typing-extensions==4.2.0 # starlette urllib3==1.26.9 # via requests -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e0fbb8092dd..fe1b8b2e131 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -90,7 +90,9 @@ botocore==1.23.24 bowler==0.9.0 # via feast (setup.py) build==0.8.0 - # via feast (setup.py) + # via + # feast (setup.py) + # pip-tools cachecontrol==0.12.11 # via firebase-admin cachetools==4.2.4 @@ -173,7 +175,7 @@ executing==0.8.3 # via stack-data fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -209,7 +211,7 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.51.0 +google-api-python-client==2.52.0 # via firebase-admin google-auth==1.35.0 # via @@ -257,8 +259,6 @@ googleapis-common-protos==1.56.3 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -269,9 +269,9 @@ grpcio==1.47.0 # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-testing==1.44.0 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.44.0 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -295,7 +295,7 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx -importlib-metadata==4.11.4 +importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest @@ -322,7 +322,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.0 +jsonschema==4.6.1 # via # altair # feast (setup.py) @@ -342,7 +342,7 @@ mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.2 +mistune==2.0.3 # via great-expectations mmh3==3.0.0 # via feast (setup.py) @@ -350,7 +350,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.14 +moto==3.1.16 # via feast (setup.py) msal==1.18.0 # via @@ -382,7 +382,7 @@ mysqlclient==2.1.1 # via feast (setup.py) nbformat==5.4.0 # via great-expectations -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit numpy==1.22.0 # via @@ -426,14 +426,12 @@ pathspec==0.9.0 pbr==5.9.0 # via mock pep517==0.12.0 - # via - # build - # pip-tools + # via build pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.6.2 +pip-tools==6.7.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -445,7 +443,7 @@ portalocker==2.4.0 # via msal-extensions pre-commit==2.19.0 # via feast (setup.py) -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.30 # via ipython proto-plus==1.20.6 # via @@ -454,7 +452,7 @@ proto-plus==1.20.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -588,7 +586,7 @@ redis==4.2.2 # via feast (setup.py) regex==2022.6.2 # via black -requests==2.28.0 +requests==2.28.1 # via # adal # adlfs @@ -667,7 +665,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -726,17 +724,17 @@ types-protobuf==3.19.22 # mypy-protobuf types-python-dateutil==2.8.18 # via feast (setup.py) -types-pytz==2022.1.0 +types-pytz==2022.1.1 # via feast (setup.py) -types-pyyaml==6.0.8 +types-pyyaml==6.0.9 # via feast (setup.py) -types-redis==4.3.2 +types-redis==4.3.3 # via feast (setup.py) -types-requests==2.27.31 +types-requests==2.28.0 # via feast (setup.py) -types-setuptools==57.4.17 +types-setuptools==57.4.18 # via feast (setup.py) -types-tabulate==0.8.10 +types-tabulate==0.8.11 # via feast (setup.py) types-urllib3==1.26.15 # via types-requests @@ -763,11 +761,11 @@ urllib3==1.26.9 # minio # requests # responses -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.14.1 +virtualenv==20.15.1 # via pre-commit volatile==2.1.0 # via bowler diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index da1a7f4345c..76e2815ed59 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -22,7 +22,7 @@ cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.0 # via requests click==8.0.1 # via @@ -40,7 +40,7 @@ dill==0.3.5.1 # via feast (setup.py) fastapi==0.78.0 # via feast (setup.py) -fastavro==1.5.1 +fastavro==1.5.2 # via # feast (setup.py) # pandavro @@ -50,15 +50,13 @@ fsspec==2022.5.0 # via dask google-api-core==2.8.2 # via feast (setup.py) -google-auth==2.8.0 +google-auth==2.9.0 # via google-api-core googleapis-common-protos==1.56.3 # via # feast (setup.py) # google-api-core # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy grpcio==1.47.0 # via # feast (setup.py) @@ -75,7 +73,7 @@ idna==3.3 # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.0 +jsonschema==4.6.1 # via feast (setup.py) locket==1.0.0 # via partd @@ -107,7 +105,7 @@ partd==1.2.0 # via dask proto-plus==1.20.6 # via feast (setup.py) -protobuf==3.19.4 +protobuf==3.20.1 # via # feast (setup.py) # google-api-core @@ -144,7 +142,7 @@ pyyaml==6.0 # dask # feast (setup.py) # uvicorn -requests==2.28.0 +requests==2.28.1 # via google-api-core rsa==4.8 # via google-auth @@ -156,7 +154,7 @@ six==1.16.0 # python-dateutil sniffio==1.2.0 # via anyio -sqlalchemy[mypy]==1.4.38 +sqlalchemy[mypy]==1.4.39 # via feast (setup.py) sqlalchemy2-stubs==0.0.2a24 # via sqlalchemy @@ -188,7 +186,7 @@ typing-extensions==4.2.0 # starlette urllib3==1.26.9 # via requests -uvicorn[standard]==0.18.1 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn diff --git a/sdk/python/tests/unit/test_proto_json.py b/sdk/python/tests/unit/test_proto_json.py index 6bfdbbbf91b..235ebc7f936 100644 --- a/sdk/python/tests/unit/test_proto_json.py +++ b/sdk/python/tests/unit/test_proto_json.py @@ -81,7 +81,7 @@ def test_feast_repeated_value(proto_json_patch): # additional structure (e.g. [1,2,3] instead of {"val": [1,2,3]}) repeated_value_str = "[1,2,3]" repeated_value_proto = RepeatedValue() - Parse(repeated_value_str, repeated_value_proto) + Parse(repeated_value_str, repeated_value_proto, "") assertpy.assert_that(len(repeated_value_proto.val)).is_equal_to(3) assertpy.assert_that(repeated_value_proto.val[0].int64_val).is_equal_to(1) assertpy.assert_that(repeated_value_proto.val[1].int64_val).is_equal_to(2) diff --git a/setup.py b/setup.py index 55a715b53af..217b2e10118 100644 --- a/setup.py +++ b/setup.py @@ -52,15 +52,15 @@ "fastavro>=1.1.0,<2", "google-api-core>=1.23.0,<3", "googleapis-common-protos>=1.52.*,<2", - "grpcio>=1.34.0,<2", - "grpcio-reflection>=1.34.0,<2", + "grpcio>=1.47.0,<2", + "grpcio-reflection>=1.47.0,<2", "Jinja2>=2,<4", "jsonschema", "mmh3", "numpy>=1.22,<2", "pandas>=1,<2", "pandavro==1.5.*", - "protobuf>=3.10,<3.20", + "protobuf>3.20,<4", "proto-plus==1.20.*", "pyarrow>=4,<7", "pydantic>=1,<2", @@ -131,8 +131,8 @@ "flake8", "black==19.10b0", "isort>=5,<6", - "grpcio-tools==1.44.0", - "grpcio-testing==1.44.0", + "grpcio-tools==1.47.0", + "grpcio-testing==1.47.0", "minio==7.1.0", "mock==2.0.0", "moto", @@ -514,8 +514,8 @@ def copy_extensions_to_source(self): use_scm_version=use_scm_version, setup_requires=[ "setuptools_scm", - "grpcio", - "grpcio-tools==1.44.0", + "grpcio==1.47.0", + "grpcio-tools==1.47.0", "mypy-protobuf==3.1", "pybindgen==0.22.0", "sphinx!=4.0.0", @@ -533,4 +533,4 @@ def copy_extensions_to_source(self): ["github.com/feast-dev/feast/go/embedded"], ) ], -) +) \ No newline at end of file From 895c2e47b144d419ac14fc62e1155521d75f2924 Mon Sep 17 00:00:00 2001 From: sfc-gh-madkins <82121043+sfc-gh-madkins@users.noreply.github.com> Date: Sat, 2 Jul 2022 13:45:24 -0500 Subject: [PATCH 05/26] fix snowflake testing (#2903) Signed-off-by: Miles Adkins --- .../tests/integration/feature_repos/repo_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 6f40d3171b0..a168f4f028f 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -78,7 +78,7 @@ "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), "redshift": ("aws", RedshiftDataSourceCreator), - "snowflake": ("aws", RedshiftDataSourceCreator), + "snowflake": ("aws", SnowflakeDataSourceCreator), } AVAILABLE_OFFLINE_STORES: List[Tuple[str, Type[DataSourceCreator]]] = [ From f78922e31a3ad78286993c7bbeca17eec23137ec Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Tue, 5 Jul 2022 10:05:10 -0700 Subject: [PATCH 06/26] chore: Change pytest fixtures to be function-scoped instead of session-scoped (#2899) * Test to trigger registry conflicts Signed-off-by: Felix Wang * Switch environment and associated fixtures to being function scoped Signed-off-by: Felix Wang * Format Signed-off-by: Felix Wang * Switch type tests to function-scoped fixtures Signed-off-by: Felix Wang * Format Signed-off-by: Felix Wang --- sdk/python/tests/conftest.py | 8 +++---- .../integration/e2e/test_go_feature_server.py | 2 +- .../registration/test_universal_types.py | 8 ++----- .../tests/unit/test_registry_conflict.py | 23 +++++++++++++++++++ 4 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 sdk/python/tests/unit/test_registry_conflict.py diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index bf69a85fa31..5fe9b5b699d 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -161,7 +161,7 @@ def start_test_local_server(repo_path: str, port: int): fs.serve("localhost", port, no_access_log=True) -@pytest.fixture(scope="session") +@pytest.fixture def environment(request, worker_id): e = construct_test_environment( request.param, worker_id=worker_id, fixture_request=request @@ -293,7 +293,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): ) -@pytest.fixture(scope="session") +@pytest.fixture def feature_server_endpoint(environment): if ( not environment.python_feature_server @@ -344,12 +344,12 @@ def _free_port(): return sock.getsockname()[1] -@pytest.fixture(scope="session") +@pytest.fixture def universal_data_sources(environment) -> TestData: return construct_universal_test_data(environment) -@pytest.fixture(scope="session") +@pytest.fixture def e2e_data_sources(environment: Environment): df = create_dataset() data_source = environment.data_source_creator.create_data_source( diff --git a/sdk/python/tests/integration/e2e/test_go_feature_server.py b/sdk/python/tests/integration/e2e/test_go_feature_server.py index 4fd003c1944..465fa41769e 100644 --- a/sdk/python/tests/integration/e2e/test_go_feature_server.py +++ b/sdk/python/tests/integration/e2e/test_go_feature_server.py @@ -35,7 +35,7 @@ ) -@pytest.fixture(scope="session") +@pytest.fixture def initialized_registry(environment, universal_data_sources): fs = environment.feature_store diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index b03303f6eef..d5cf270b252 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -64,9 +64,7 @@ class TypeTestConfig: @pytest.fixture( - params=OFFLINE_TYPE_TEST_CONFIGS, - scope="session", - ids=[str(c) for c in OFFLINE_TYPE_TEST_CONFIGS], + params=OFFLINE_TYPE_TEST_CONFIGS, ids=[str(c) for c in OFFLINE_TYPE_TEST_CONFIGS], ) def offline_types_test_fixtures(request, environment): config: TypeTestConfig = request.param @@ -80,9 +78,7 @@ def offline_types_test_fixtures(request, environment): @pytest.fixture( - params=ONLINE_TYPE_TEST_CONFIGS, - scope="session", - ids=[str(c) for c in ONLINE_TYPE_TEST_CONFIGS], + params=ONLINE_TYPE_TEST_CONFIGS, ids=[str(c) for c in ONLINE_TYPE_TEST_CONFIGS], ) def online_types_test_fixtures(request, environment): return get_fixtures(request, environment) diff --git a/sdk/python/tests/unit/test_registry_conflict.py b/sdk/python/tests/unit/test_registry_conflict.py new file mode 100644 index 00000000000..12f666556a1 --- /dev/null +++ b/sdk/python/tests/unit/test_registry_conflict.py @@ -0,0 +1,23 @@ +import pytest + +from feast.entity import Entity + + +@pytest.mark.integration +def test_apply_first_entity(environment): + entity = Entity(name="first") + fs = environment.feature_store + fs.apply([entity]) + + entities = fs.list_entities() + assert len(entities) == 1 + + +@pytest.mark.integration +def test_apply_second_entity(environment): + entity = Entity(name="second") + fs = environment.feature_store + fs.apply([entity]) + + entities = fs.list_entities() + assert len(entities) == 1 From 568058aee02e4e8b961caf312e3f1c7ff909a3f2 Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:06:08 +0200 Subject: [PATCH 07/26] fix: Change the feature store plan method to public modifier (#2904) Signed-off-by: Breno Costa --- sdk/python/feast/feature_store.py | 8 ++++---- sdk/python/feast/repo_operations.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index de52b9e3f3e..ce72c3c8034 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -511,8 +511,8 @@ def _get_features( return _feature_refs def _should_use_plan(self): - """Returns True if _plan and _apply_diffs should be used, False otherwise.""" - # Currently only the local provider with sqlite online store supports _plan and _apply_diffs. + """Returns True if plan and _apply_diffs should be used, False otherwise.""" + # Currently only the local provider with sqlite online store supports plan and _apply_diffs. return self.config.provider == "local" and ( self.config.online_store and self.config.online_store.type == "sqlite" ) @@ -636,7 +636,7 @@ def _get_feature_views_to_materialize( return feature_views_to_materialize @log_exceptions_and_usage - def _plan( + def plan( self, desired_repo_contents: RepoContents ) -> Tuple[RegistryDiff, InfraDiff, Infra]: """Dry-run registering objects to metadata store. @@ -670,7 +670,7 @@ def _plan( ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) - >>> registry_diff, infra_diff, new_infra = fs._plan(RepoContents( + >>> registry_diff, infra_diff, new_infra = fs.plan(RepoContents( ... data_sources=[driver_hourly_stats], ... feature_views=[driver_hourly_stats_view], ... on_demand_feature_views=list(), diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 37daa6500eb..9a5e64f8c30 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -183,7 +183,7 @@ def plan(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool) for data_source in data_sources: data_source.validate(store.config) - registry_diff, infra_diff, _ = store._plan(repo) + registry_diff, infra_diff, _ = store.plan(repo) click.echo(registry_diff.to_string()) click.echo(infra_diff.to_string()) @@ -262,7 +262,7 @@ def apply_total_with_repo_instance( for data_source in data_sources: data_source.validate(store.config) - registry_diff, infra_diff, new_infra = store._plan(repo) + registry_diff, infra_diff, new_infra = store.plan(repo) # For each object in the registry, determine whether it should be kept or deleted. ( From d36bf65ee0ec972dfcc03bee87d7a664262ef319 Mon Sep 17 00:00:00 2001 From: Prasad Zende <37083921+prasadzende@users.noreply.github.com> Date: Wed, 6 Jul 2022 21:18:12 +0530 Subject: [PATCH 08/26] docs: Grammatically updated the quickstart guide docs (#2913) Signed-off-by: Prasad --- docs/getting-started/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index b5fe7bad4b9..972ffa13a9c 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -224,7 +224,7 @@ To train a model, we need features and labels. Often, this label data is stored The user can query that table of labels with timestamps and pass that into Feast as an _entity dataframe_ for training data generation. In many cases, Feast will also intelligently join relevant tables to create the relevant feature vectors. -* Note that we include timestamps because want the features for the same driver at various timestamps to be used in a model. +* Note that we include timestamps because we want the features for the same driver at various timestamps to be used in a model. {% tabs %} {% tab title="Python" %} From b5b70549646ef1eae6906105d4ecbcfdb5cd49f9 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 6 Jul 2022 11:38:13 -0700 Subject: [PATCH 09/26] chore: Refactor StreamFeatureViewMeta to FeatureViewMeta and dedupe (#2915) * chore: Refactor StreamFeatureViewMeta to FeatureViewMeta and deduplicate code Signed-off-by: Achal Shah * ttl_duration Signed-off-by: Achal Shah * nits Signed-off-by: Achal Shah --- protos/feast/core/StreamFeatureView.proto | 12 +------- sdk/python/feast/feature_view.py | 37 ++++++++++++++--------- sdk/python/feast/stream_feature_view.py | 26 ++-------------- 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/protos/feast/core/StreamFeatureView.proto b/protos/feast/core/StreamFeatureView.proto index d217b86a3f4..06e9ee0612f 100644 --- a/protos/feast/core/StreamFeatureView.proto +++ b/protos/feast/core/StreamFeatureView.proto @@ -34,7 +34,7 @@ import "feast/core/Aggregation.proto"; message StreamFeatureView { // User-specified specifications of this feature view. StreamFeatureViewSpec spec = 1; - StreamFeatureViewMeta meta = 2; + FeatureViewMeta meta = 2; } // Next available id: 17 @@ -90,13 +90,3 @@ message StreamFeatureViewSpec { string timestamp_field = 16; } -message StreamFeatureViewMeta { - // Time where this Feature View is created - google.protobuf.Timestamp created_timestamp = 1; - - // Time where this Feature View is last updated - google.protobuf.Timestamp last_updated_timestamp = 2; - - // List of pairs (start_time, end_time) for which this feature view has been materialized. - repeated MaterializationInterval materialization_intervals = 3; -} diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 348c3019c5b..dd8cb4f0a67 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -407,21 +407,8 @@ def to_proto(self) -> FeatureViewProto: Returns: A FeatureViewProto protobuf. """ - meta = FeatureViewMetaProto(materialization_intervals=[]) - if self.created_timestamp: - meta.created_timestamp.FromDatetime(self.created_timestamp) - if self.last_updated_timestamp: - meta.last_updated_timestamp.FromDatetime(self.last_updated_timestamp) - for interval in self.materialization_intervals: - interval_proto = MaterializationIntervalProto() - interval_proto.start_time.FromDatetime(interval[0]) - interval_proto.end_time.FromDatetime(interval[1]) - meta.materialization_intervals.append(interval_proto) - - ttl_duration = None - if self.ttl is not None: - ttl_duration = Duration() - ttl_duration.FromTimedelta(self.ttl) + meta = self.to_proto_meta() + ttl_duration = self.get_ttl_duration() batch_source_proto = self.batch_source.to_proto() batch_source_proto.data_source_class_type = f"{self.batch_source.__class__.__module__}.{self.batch_source.__class__.__name__}" @@ -447,6 +434,26 @@ def to_proto(self) -> FeatureViewProto: return FeatureViewProto(spec=spec, meta=meta) + def to_proto_meta(self): + meta = FeatureViewMetaProto(materialization_intervals=[]) + if self.created_timestamp: + meta.created_timestamp.FromDatetime(self.created_timestamp) + if self.last_updated_timestamp: + meta.last_updated_timestamp.FromDatetime(self.last_updated_timestamp) + for interval in self.materialization_intervals: + interval_proto = MaterializationIntervalProto() + interval_proto.start_time.FromDatetime(interval[0]) + interval_proto.end_time.FromDatetime(interval[1]) + meta.materialization_intervals.append(interval_proto) + return meta + + def get_ttl_duration(self): + ttl_duration = None + if self.ttl is not None: + ttl_duration = Duration() + ttl_duration.FromTimedelta(self.ttl) + return ttl_duration + @classmethod def from_proto(cls, feature_view_proto: FeatureViewProto): """ diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 077d8ab89a5..f19b1fcff7e 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -6,7 +6,6 @@ from typing import Dict, List, Optional, Tuple, Union import dill -from google.protobuf.duration_pb2 import Duration from typeguard import typechecked from feast import utils @@ -16,18 +15,12 @@ from feast.feature_view import FeatureView from feast.field import Field from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto -from feast.protos.feast.core.FeatureView_pb2 import ( - MaterializationInterval as MaterializationIntervalProto, -) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( UserDefinedFunction as UserDefinedFunctionProto, ) from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureView as StreamFeatureViewProto, ) -from feast.protos.feast.core.StreamFeatureView_pb2 import ( - StreamFeatureViewMeta as StreamFeatureViewMetaProto, -) from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureViewSpec as StreamFeatureViewSpecProto, ) @@ -170,23 +163,8 @@ def __hash__(self) -> int: return super().__hash__() def to_proto(self): - meta = StreamFeatureViewMetaProto(materialization_intervals=[]) - if self.created_timestamp: - meta.created_timestamp.FromDatetime(self.created_timestamp) - - if self.last_updated_timestamp: - meta.last_updated_timestamp.FromDatetime(self.last_updated_timestamp) - - for interval in self.materialization_intervals: - interval_proto = MaterializationIntervalProto() - interval_proto.start_time.FromDatetime(interval[0]) - interval_proto.end_time.FromDatetime(interval[1]) - meta.materialization_intervals.append(interval_proto) - - ttl_duration = None - if self.ttl is not None: - ttl_duration = Duration() - ttl_duration.FromTimedelta(self.ttl) + meta = self.to_proto_meta() + ttl_duration = self.get_ttl_duration() batch_source_proto = None if self.batch_source: From b7ec8c17dfb281d8ea524374c9c847fbe7c753cd Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 8 Jul 2022 14:35:36 -0400 Subject: [PATCH 10/26] chore: Update Feast UI dependency to reflect recent changes (#2924) chore: Update Feast UI dependency Signed-off-by: Danny Chiao --- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 556637aaae2..ec046557236 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^57.0.0", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "^0.20.4", + "@feast-dev/feast-ui": "^0.20.5", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index f6301957c82..83157394caf 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1345,10 +1345,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@^0.20.4": - version "0.20.4" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.20.4.tgz#4b918f8922f3eecd9e3e7323f25ba9cac78a4567" - integrity sha512-KTUhKni7t++G6UwXyPbGWXwWHnTOVTH8ouYCoHXbGorgRL3K4fbq5tCSCJzP9L5FAo+cF1AjVZNRgwzPe6vAgA== +"@feast-dev/feast-ui@^0.20.5": + version "0.20.5" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.20.5.tgz#bb0d6fc81cbd92ca69b779982ab151a8d9cabaee" + integrity sha512-BwMPJSv1MkylHxPnU/2fZX77AC/G4H2DIf+HAj80ZklwB0zbmeZzhXFrVh4xSheevGZFh0L839JeL14WfXPZsA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index 252faf8613d..22128cc9681 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.20.4", + "version": "0.20.5", "private": false, "files": [ "dist" From 5167f607d30473779cefd4d79e4e72bbfde53661 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:31:39 -0700 Subject: [PATCH 11/26] chore: Bump moment from 2.29.2 to 2.29.4 in /ui (#2926) chore(deps): bump moment from 2.29.2 to 2.29.4 in /ui Bumps [moment](https://github.com/moment/moment) from 2.29.2 to 2.29.4. - [Release notes](https://github.com/moment/moment/releases) - [Changelog](https://github.com/moment/moment/blob/develop/CHANGELOG.md) - [Commits](https://github.com/moment/moment/compare/2.29.2...2.29.4) --- updated-dependencies: - dependency-name: moment dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 998565a77ac..78c069b38ab 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -7326,9 +7326,9 @@ mkdirp@^0.5.5, mkdirp@~0.5.1: minimist "^1.2.5" moment@^2.29.1: - version "2.29.2" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4" - integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg== + version "2.29.4" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== ms@2.0.0: version "2.0.0" From f4f4894d1db4b2192a502999f87711b0529e6e83 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 15:21:38 -0700 Subject: [PATCH 12/26] fix: Fix the go build and use CgoArrowAllocator to prevent incorrect garbage collection (#2919) * Temp fix Signed-off-by: Kevin Zhang * Temp fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * add dynamic linking flags Signed-off-by: Kevin Zhang * Update gitignore Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix workflows Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 12 ++++++++++ .github/workflows/linter.yml | 18 ++++++++++++-- .github/workflows/master_only.yml | 12 ++++++++++ .github/workflows/pr_integration_tests.yml | 12 ++++++++++ .github/workflows/unit_tests.yml | 24 +++++++++++++++---- Makefile | 19 ++++++++------- .../feature-servers/go-feature-retrieval.md | 16 +++++++++++-- go.mod | 8 +++---- go.sum | 9 ++++--- go/embedded/online_features.go | 2 +- .../onlinestore/sqliteonlinestore_test.go | 4 ++-- setup.py | 5 +++- 12 files changed, 114 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ea93cc85a44..c69ba4687d8 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -62,6 +62,18 @@ jobs: registry-url: 'https://registry.npmjs.org' - name: Build UI run: make build-ui + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Build wheels uses: pypa/cibuildwheel@v2.7.0 env: diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a0a6d7dd382..ba475e2585b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -39,6 +39,14 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install dependencies run: | make compile-protos-go @@ -63,7 +71,13 @@ jobs: - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<22.1" - - name: Install dependencies - run: make install-go-proto-dependencies + - name: Install apache-arrow on ubuntu + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Lint go run: make lint-go \ No newline at end of file diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 0cb49bb525c..c9ebcdaf040 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -127,6 +127,18 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index e1c7ed2de2b..db9e48fc2d8 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -153,6 +153,18 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a9bf3deba32..b3fadb21215 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -54,6 +54,18 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python @@ -92,9 +104,13 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - - name: Install dependencies - run: make install-go-proto-dependencies - - name: Compile protos - run: make compile-protos-go + - name: Install apache-arrow on ubuntu + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Test run: make test-go diff --git a/Makefile b/Makefile index 176e2cb3545..4b541963af9 100644 --- a/Makefile +++ b/Makefile @@ -172,12 +172,12 @@ install-go-proto-dependencies: install-go-ci-dependencies: # TODO: currently gopy installation doesn't work w/o explicit go get in the next line # TODO: there should be a better way to install gopy - go get github.com/go-python/gopy@v0.4.0 + go get github.com/go-python/gopy@v0.4.4 go install golang.org/x/tools/cmd/goimports # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` # but the following command is needed to install it for some reason. go install github.com/go-python/gopy - python -m pip install pybindgen==0.22.0 + python -m pip install pybindgen==0.22.0 protobuf==3.20.1 install-protoc-dependencies: pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 @@ -186,18 +186,21 @@ compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - COMPILE_GO=True python setup.py build_ext --inplace + CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace -# Needs feast package to setup the feature store -test-go: compile-protos-go +install-feast-ci-locally: pip install -e ".[ci]" - go test ./... + +# Needs feast package to setup the feature store +# CGO flag is due to this issue: https://github.com/golang/go/wiki/InvalidFlag +test-go: compile-protos-go compile-go-lib install-feast-ci-locally + CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... format-go: gofmt -s -w go/ -lint-go: compile-protos-go - go vet ./go/internal/feast ./go/embedded +lint-go: compile-protos-go compile-go-lib + go vet -tags cgo,ccalloc ./go/internal/feast ./go/embedded # Docker diff --git a/docs/reference/feature-servers/go-feature-retrieval.md b/docs/reference/feature-servers/go-feature-retrieval.md index 685e7201cb6..92a9ca2ebe3 100644 --- a/docs/reference/feature-servers/go-feature-retrieval.md +++ b/docs/reference/feature-servers/go-feature-retrieval.md @@ -16,8 +16,20 @@ However, some additional dependencies are required for Go <-> Python interoperab ``` pip install feast[go] ``` +You will also have to install the apache-arrow c++ libraries, since we use the cgo memory allocator to prevent memory from being incorrectly garbage collected, detailed in these [docs](https://pkg.go.dev/github.com/apache/arrow/go/arrow@v0.0.0-20211112161151-bc219186db40/cdata#ExportArrowRecordBatch). -For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. +For macos, run `brew install apache-arrow`. +For linux users, you have to install `libarrow-dev`. +``` +sudo apt update +sudo apt install -y -V ca-certificates lsb-release wget +wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +sudo apt update +sudo apt install -y -V libarrow-dev # For C++ +``` + +For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. In order to build the go binaries, you will need to install the `apache-arrow` c++ libraries. ## Usage @@ -63,7 +75,7 @@ feature_server: emit_timeout_micro_secs: 10000 queue_capacity: 10000 ``` -All these parameters are optional. +All these parameters are optional. ## Future/Current Work diff --git a/go.mod b/go.mod index 90ddb93e21c..ef64636e73e 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,10 @@ module github.com/feast-dev/feast go 1.17 -replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 - require ( github.com/apache/arrow/go/v8 v8.0.0 github.com/ghodss/yaml v1.0.0 - github.com/go-python/gopy v0.4.0 + github.com/go-python/gopy v0.4.4 github.com/go-redis/redis/v8 v8.11.4 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 @@ -28,6 +26,8 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/gonuts/commander v0.1.0 // indirect + github.com/gonuts/flag v0.1.0 // indirect github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.1 // indirect @@ -38,7 +38,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 933ecf6b29c..d0b7f8fcd80 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 h1:Go714ObVP1O+a6qK7haXVL28QNm6WMD8bwnN9EA8PlM= -github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0/go.mod h1:ZO6vpitQ61NVoQP/2yOubPS6ET5pP3CAWCiMYn5eqCc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -112,6 +110,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-python/gopy v0.4.4 h1:3LTsrfVcmg2VEM6wU+eh4d9EZn5H2iogObXjiQHrF8Q= +github.com/go-python/gopy v0.4.4/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -147,7 +147,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= +github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -439,8 +441,9 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index f6b21169e10..e5860507e45 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -180,7 +180,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures( outputFields := make([]arrow.Field, 0) outputColumns := make([]arrow.Array, 0) - pool := memory.NewGoAllocator() + pool := memory.NewCgoArrowAllocator() for _, featureVector := range resp { outputFields = append(outputFields, arrow.Field{ diff --git a/go/internal/feast/onlinestore/sqliteonlinestore_test.go b/go/internal/feast/onlinestore/sqliteonlinestore_test.go index 5af1c1f4ce4..e5e6e85e56c 100644 --- a/go/internal/feast/onlinestore/sqliteonlinestore_test.go +++ b/go/internal/feast/onlinestore/sqliteonlinestore_test.go @@ -17,9 +17,9 @@ import ( func TestSqliteAndFeatureRepoSetup(t *testing.T) { dir := t.TempDir() feature_repo_path := filepath.Join(dir, "feature_repo") + err := test.SetupCleanFeatureRepo(dir) assert.Nil(t, err) - config, err := registry.NewRepoConfigFromFile(feature_repo_path) assert.Nil(t, err) assert.Equal(t, "feature_repo", config.Project) @@ -37,9 +37,9 @@ func TestSqliteOnlineRead(t *testing.T) { dir := t.TempDir() feature_repo_path := filepath.Join(dir, "feature_repo") test.SetupCleanFeatureRepo(dir) - config, err := registry.NewRepoConfigFromFile(feature_repo_path) assert.Nil(t, err) + store, err := NewSqliteOnlineStore("feature_repo", config, config.OnlineStore) defer store.Destruct() assert.Nil(t, err) diff --git a/setup.py b/setup.py index 217b2e10118..deaf47f955f 100644 --- a/setup.py +++ b/setup.py @@ -430,7 +430,7 @@ def build_extension(self, ext: Extension): destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"], env={"PATH": bin_path, **go_env}) - subprocess.check_call(["go", "get", "github.com/go-python/gopy@v0.4.0"], + subprocess.check_call(["go", "get", "github.com/go-python/gopy@v0.4.4"], env={"PATH": bin_path, **go_env}) subprocess.check_call(["go", "install", "github.com/go-python/gopy"], env={"PATH": bin_path, **go_env}) @@ -442,6 +442,9 @@ def build_extension(self, ext: Extension): destination, "-vm", sys.executable, + "--build-tags", + 'cgo,ccalloc', + "--dynamic-link=True", "-no-make", *ext.sources, ], From 935b34c84941320dbd251240c91a76f4584a00ec Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Mon, 11 Jul 2022 13:28:08 -0400 Subject: [PATCH 13/26] ci: Fixing local integration tests, defaulting to test containers (#2927) --- .../workflows/pr_local_integration_tests.yml | 63 +++++++++++++++++++ CONTRIBUTING.md | 24 ++++--- Makefile | 25 ++++++-- sdk/python/tests/conftest.py | 5 +- .../feature_repos/repo_configuration.py | 2 + .../universal/online_store/datastore.py | 2 +- .../universal/online_store/dynamodb.py | 2 +- .../universal/online_store/hbase.py | 2 +- .../universal/online_store/redis.py | 2 +- .../integration/registration/test_registry.py | 11 ++++ 10 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/pr_local_integration_tests.yml diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml new file mode 100644 index 00000000000..0736ae29dc3 --- /dev/null +++ b/.github/workflows/pr_local_integration_tests.yml @@ -0,0 +1,63 @@ +name: pr-local-integration-tests +# This runs local tests with containerized stubs of online stores. This is the main dev workflow + +on: + pull_request_target: + types: + - opened + - synchronize + - labeled + +jobs: + integration-test-python-local: + # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: + (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm'))) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.8" ] + os: [ ubuntu-latest ] + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v2 + with: + # pull_request_target runs the workflow in the context of the base repo + # as such actions/checkout needs to be explicit configured to retrieve + # code from the PR. + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v2 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<22.1" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.pip-cache.outputs.dir }} + /opt/hostedtoolcache/Python + /Users/runner/hostedtoolcache/Python + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + restore-keys: | + ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Install pip-tools + run: pip install pip-tools + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Test local integration tests + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + run: make test-python-integration-local diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4bd14d762a5..9c25a835bdd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,17 +133,19 @@ make test-python ### Integration Tests There are two sets of tests you can run: -1. Local integration tests (for faster development) +1. Local integration tests (for faster development, tests file offline store & key online stores) 2. Full integration tests (requires cloud environment setups) #### Local integration tests -To get local integration tests running, you'll need to have Redis setup: +For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/) -Redis -1. Install Redis: [Quickstart](https://redis.io/topics/quickstart) -2. Run `redis-server` +It leverages a file based offline store to test against emulated versions of Datastore, DynamoDB, and Redis, using ephemeral containers. -Now run `make test-python-universal-local` +These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down. + +```sh +make test-python-integration-local +``` #### Full integration tests To test across clouds, on top of setting up Redis, you also need GCP / AWS / Snowflake setup. @@ -166,7 +168,15 @@ To test across clouds, on top of setting up Redis, you also need GCP / AWS / Sno 2. Modify `RedshiftDataSourceCreator` to use your credentials **Snowflake** -- See https://signup.snowflake.com/ +1. See https://signup.snowflake.com/ to setup a trial. +2. Then to run successfully, you'll need some environment variables setup: +```sh +export SNOWFLAKE_CI_DEPLOYMENT='[snowflake_deployment]' +export SNOWFLAKE_CI_USER='[your user]' +export SNOWFLAKE_CI_PASSWORD='[your pw]' +export SNOWFLAKE_CI_ROLE='[your CI role e.g. SYSADMIN]' +export SNOWFLAKE_CI_WAREHOUSE='[your warehouse]' +``` Then run `make test-python-integration`. Note that for Snowflake / GCP / AWS, this will create new temporary tables / datasets. diff --git a/Makefile b/Makefile index 4b541963af9..6d733ac61f9 100644 --- a/Makefile +++ b/Makefile @@ -68,8 +68,26 @@ test-python: test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests +test-python-integration-local: + @(docker info > /dev/null 2>&1 && \ + FEAST_USAGE=False \ + IS_TEST=True \ + FEAST_IS_LOCAL_TEST=True \ + FEAST_LOCAL_ONLINE_CONTAINER=True \ + python -m pytest -n 8 --integration \ + -k "not test_apply_entity_integration and \ + not test_apply_feature_view_integration and \ + not test_apply_data_source_integration" \ + sdk/python/tests \ + ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; + test-python-integration-container: - FEAST_USAGE=False IS_TEST=True FEAST_LOCAL_ONLINE_CONTAINER=True python -m pytest -n 8 --integration sdk/python/tests + @(docker info > /dev/null 2>&1 && \ + FEAST_USAGE=False \ + IS_TEST=True \ + FEAST_LOCAL_ONLINE_CONTAINER=True \ + python -m pytest -n 8 --integration sdk/python/tests \ + ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; test-python-universal-contrib: PYTHONPATH='.' \ @@ -104,14 +122,11 @@ test-python-universal-postgres: not test_universal_types" \ sdk/python/tests -test-python-universal-local: - FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration sdk/python/tests - test-python-universal: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests test-python-go-server: compile-go-lib - FEAST_USAGE=False IS_TEST=True FEAST_GO_FEATURE_RETRIEVAL=True pytest --integration --goserver sdk/python/tests + FEAST_USAGE=False IS_TEST=True pytest --integration --goserver sdk/python/tests format-python: # Sort diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 5fe9b5b699d..35067317cf1 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -110,7 +110,10 @@ def pytest_collection_modifyitems(config, items: List[Item]): items.append(t) goserver_tests = [t for t in items if "goserver" in t.keywords] - if should_run_goserver: + if not should_run_goserver: + for t in goserver_tests: + items.remove(t) + else: items.clear() for t in goserver_tests: items.append(t) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index a168f4f028f..4dc1db4a135 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -91,6 +91,7 @@ "sqlite": ({"type": "sqlite"}, None), } +# Only configure Cloud DWH if running full integration tests if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True": AVAILABLE_OFFLINE_STORES.extend( [ @@ -141,6 +142,7 @@ } +# Replace online stores with emulated online stores if we're running local integration tests if os.getenv("FEAST_LOCAL_ONLINE_CONTAINER", "False").lower() == "true": replacements: Dict[ str, Tuple[Union[str, Dict[str, str]], Optional[Type[OnlineStoreCreator]]] diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/datastore.py b/sdk/python/tests/integration/feature_repos/universal/online_store/datastore.py index 6067a1ff4b8..b5bbb94f7c1 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/datastore.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/datastore.py @@ -27,7 +27,7 @@ def create_online_store(self) -> Dict[str, str]: self.container.start() log_string_to_wait_for = r"\[datastore\] Dev App Server is now running" wait_for_logs( - container=self.container, predicate=log_string_to_wait_for, timeout=5 + container=self.container, predicate=log_string_to_wait_for, timeout=10 ) exposed_port = self.container.get_exposed_port("8081") os.environ[datastore.client.DATASTORE_EMULATOR_HOST] = f"0.0.0.0:{exposed_port}" diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py index 473b7acee97..1aefdffb24b 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py @@ -21,7 +21,7 @@ def create_online_store(self) -> Dict[str, str]: "Initializing DynamoDB Local with the following configuration:" ) wait_for_logs( - container=self.container, predicate=log_string_to_wait_for, timeout=5 + container=self.container, predicate=log_string_to_wait_for, timeout=10 ) exposed_port = self.container.get_exposed_port("8000") return { diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/hbase.py b/sdk/python/tests/integration/feature_repos/universal/online_store/hbase.py index ecaace87097..dba611b30bc 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/hbase.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/hbase.py @@ -19,7 +19,7 @@ def create_online_store(self) -> Dict[str, str]: "Initializing Hbase Local with the following configuration:" ) wait_for_logs( - container=self.container, predicate=log_string_to_wait_for, timeout=5 + container=self.container, predicate=log_string_to_wait_for, timeout=10 ) exposed_port = self.container.get_exposed_port("9090") return {"type": "hbase", "host": "127.0.0.1", "port": exposed_port} diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py b/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py index 49951876652..11d62d9d30a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py @@ -17,7 +17,7 @@ def create_online_store(self) -> Dict[str, str]: self.container.start() log_string_to_wait_for = "Ready to accept connections" wait_for_logs( - container=self.container, predicate=log_string_to_wait_for, timeout=5 + container=self.container, predicate=log_string_to_wait_for, timeout=10 ) exposed_port = self.container.get_exposed_port("6379") return {"type": "redis", "connection_string": f"localhost:{exposed_port},db=0"} diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 36e19e222ac..ac7696f6e7f 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -571,7 +571,18 @@ def test_apply_feature_view_integration(test_registry): @pytest.mark.parametrize( "test_registry", [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")], ) +def test_apply_data_source_integration(test_registry: Registry): + run_test_data_source_apply(test_registry) + + +@pytest.mark.parametrize( + "test_registry", [lazy_fixture("local_registry")], +) def test_apply_data_source(test_registry: Registry): + run_test_data_source_apply(test_registry) + + +def run_test_data_source_apply(test_registry: Registry): # Create Feature Views batch_source = FileSource( name="test_source", From 4b69e0e839e4fdc6268ef077f02bd5b727c44370 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 13:55:22 -0700 Subject: [PATCH 14/26] fix: Fix build wheels workflow to install apache-arrow correctly (#2932) * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 18 ++++++------------ .../workflows/pr_local_integration_tests.yml | 9 +++++++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c69ba4687d8..1b0059f1a31 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -62,18 +62,6 @@ jobs: registry-url: 'https://registry.npmjs.org' - name: Build UI run: make build-ui - - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' - run: brew install apache-arrow - name: Build wheels uses: pypa/cibuildwheel@v2.7.0 env: @@ -86,7 +74,12 @@ jobs: curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz tar -C /usr/local -xzf go.tar.gz go version + yum -y update && + yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm && + yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm && + yum install -y --enablerepo=epel arrow-devel # For C++ CIBW_BEFORE_ALL_MACOS: | + brew install apache-arrow curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg sudo installer -pkg python.pkg -target / # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. @@ -111,6 +104,7 @@ jobs: CIBW_BEFORE_BUILD: | git status git restore go.mod go.sum + brew install apache-arrow - uses: actions/upload-artifact@v2 with: name: wheels diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 0736ae29dc3..08470833936 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -56,6 +56,15 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install dependencies run: make install-python-ci-dependencies - name: Test local integration tests From 3650c4a54d5236c01ef03139c927de78831aeac4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 17:12:00 -0700 Subject: [PATCH 15/26] chore: Update docs with new release workflow (#2898) * Update docs Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- docs/project/release-process.md | 81 ++++++++++----------------------- 1 file changed, 24 insertions(+), 57 deletions(-) diff --git a/docs/project/release-process.md b/docs/project/release-process.md index af573c92c76..f7102d13c39 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -4,60 +4,27 @@ For Feast maintainers, these are the concrete steps for making a new release. -1. For new major or minor release, create and check out the release branch for the new stream, e.g. `v0.6-branch`. For a patch version, check out the stream's release branch. -2. Update the [CHANGELOG.md](../../CHANGELOG.md). See the [Creating a change log](release-process.md#creating-a-change-log) guide and commit - * Make to review each PR in the changelog to [flag any breaking changes and deprecation.](release-process.md#flag-breaking-changes-and-deprecations) -3. Update versions for the release/release candidate with a commit: - 1. In the root `pom.xml`, remove `-SNAPSHOT` from the `` property, update versions, and commit. - 2. Tag the commit with the release version, using a `v` and `sdk/go/v` prefixes - * for a release candidate, create tags `vX.Y.Z-rc.N`and `sdk/go/vX.Y.Z-rc.N` - * for a stable release `X.Y.Z` create tags `vX.Y.Z` and `sdk/go/vX.Y.Z` - 3. Check that versions are updated with `make lint-versions`. - 4. If changes required are flagged by the version lint, make the changes, amend the commit and move the tag to the new commit. -4. Push the commits and tags. Make sure the CI passes. - * If the CI does not pass, or if there are new patches for the release fix, repeat step 2 & 3 with release candidates until stable release is achieved. -5. Bump to the next patch version in the release branch, append `-SNAPSHOT` in `pom.xml` and push. -6. Create a PR against master to: - 1. Bump to the next major/minor version and append `-SNAPSHOT` . - 2. Add the change log by applying the change log commit created in step 2. - 3. Check that versions are updated with `env TARGET_MERGE_BRANCH=master make lint-versions` -7. Create a [GitHub release](https://github.com/feast-dev/feast/releases) which includes a summary of im~~p~~ortant changes as well as any artifacts associated with the release. Make sure to include the same change log as added in [CHANGELOG.md](../../CHANGELOG.md). Use `Feast vX.Y.Z` as the title. - -When a tag that matches a Semantic Version string is pushed, CI will automatically build and push the relevant artifacts to their repositories or package managers (docker images, Python wheels, etc). JVM artifacts are promoted from Sonatype OSSRH to Maven Central, but it sometimes takes some time for them to be available. The `sdk/go/v tag` is required to version the Go SDK go module so that users can go get a specific tagged release of the Go SDK. - -### Creating a change log - -We use an [open source change log generator](https://hub.docker.com/r/ferrarimarco/github-changelog-generator/) to generate change logs. The process still requires a little bit of manual effort. - -1. Create a GitHub token as [per these instructions](https://github.com/github-changelog-generator/github-changelog-generator#github-token). The token is used as an input argument (`-t`) to the change log generator. -2. The change log generator configuration below will look for unreleased changes on a specific branch. The branch will be `master` for a major/minor release, or a release branch (`v0.4-branch`) for a patch release. You will need to set the branch using the `--release-branch` argument. -3. You should also set the `--future-release` argument. This is the version you are releasing. The version can still be changed at a later date. -4. Update the arguments below and run the command to generate the change log to the console. - -``` -docker run -it --rm ferrarimarco/github-changelog-generator \ ---user feast-dev \ ---project feast \ ---release-branch \ ---future-release \ ---unreleased-only \ ---no-issues \ ---bug-labels kind/bug \ ---enhancement-labels kind/feature \ ---breaking-labels compat/breaking \ --t \ ---max-issues 1 \ --o -``` - -1. Review each change log item. - * Make sure that sentences are grammatically correct and well formatted (although we will try to enforce this at the PR review stage). - * Make sure that each item is categorised correctly. You will see the following categories: `Breaking changes`, `Implemented enhancements`, `Fixed bugs`, and `Merged pull requests`. Any unlabelled PRs will be found in `Merged pull requests`. It's important to make sure that any `breaking changes`, `enhancements`, or `bug fixes` are pulled up out of `merged pull requests` into the correct category. Housekeeping, tech debt clearing, infra changes, or refactoring do not count as `enhancements`. Only enhancements a user benefits from should be listed in that category. - * Make sure that the "Full Change log" link is actually comparing the correct tags (normally your released version against the previously version). - * Make sure that release notes and breaking changes are present. - -### Flag Breaking Changes & Deprecations - -It's important to flag breaking changes and deprecation to the API for each release so that we can maintain API compatibility. - -Developers should have flagged PRs with breaking changes with the `compat/breaking` label. However, it's important to double check each PR's release notes and contents for changes that will break API compatibility and manually label `compat/breaking` to PRs with undeclared breaking changes. The change log will have to be regenerated if any new labels have to be added. +### Pre-release Verification (Verification that wheels are built correctly) for minor release. +1. Merge upstream master changes into your **fork**. Make sure you are running the workflow off of your fork! +2. Create a tag manually for the release on your fork. For example, if you are doing a release for version 0.22.0, create a tag by doing the following. + - Checkout master branch and run `git tag v0.22.0`. + - Run `git push --tags` to push the tag to your forks master branch. +3. Access the `Actions` tab on your github UI on your fork and click the `build_wheels` action. This workflow will build the python sdk wheels for Python 3.8-3.10 on MacOS 10.15 and Linux and verify that these wheels are correct. The publish workflow uses this action to publish the python wheels for a new release to pypi. +4. Look for the header `This workflow has a workflow_dispatch event trigger` and click `Run Workflow` on the right. +5. Run the workflow off of the tag you just created(`v0.22.0` in this case) and verify that the workflow worked (i.e ensure that all jobs are green). + +### Pre-release Verification (Verification that wheels are built correctly) for patch release. +1. Check out the branch of your release (e.g `v0.22-branch` on your local **fork**) and push this to your fork (`git push -u origin `). +2. Cherry pick commits that are relevant to the patch release onto your forked branch. +3. Checkout the release branch and add a patch release tag (e.g `v0.22.1`) by running `git tag `. +4. Push tags to your origin branch with `git push origin `. +5. Kick off `build_wheels` workflow in the same way as is detailed in the last section on of the patch release tag. + +### Release for Python and Java SDK +1. Generate a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) or retrieve your saved personal access token. + - The personal access token should have all of the permissions under the `repo` checkbox. +2. Access the `Actions` tab on the main `feast-dev/feast` repo and find the `release` action. +3. Look for the header `This workflow has a workflow_dispatch event trigger` again and click `Run Workflow` on the right. +4. Try the dry run first with your personal access token. If this succeeds, uncheck `Dry Run` and run the release workflow. +5. All of the jobs should succeed besides the UI job which needs to be released separately. Ping a maintainer on Slack to run the UI release manually. +6. Try to install the feast release in your local environment and test out the `feast init` -> `feast apply` workflow to verify as a sanity check that the release worked correctly. From 9840c1b4c64898fcb06e34dcea14becce8af44c3 Mon Sep 17 00:00:00 2001 From: Hai Nguyen Date: Tue, 12 Jul 2022 11:59:06 +0700 Subject: [PATCH 16/26] fix: Resolve small typo in README file (#2930) Fix small type in README file Signed-off-by: Hai Nguyen --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0f77fbd42ca..6bd7c95b7dc 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ The list below contains the functionality that contributors are planning to deve * [ ] Java Client * [ ] Go Client * [ ] Delete API - * [] Feature Logging (for training) + * [ ] Feature Logging (for training) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * [ ] Training-serving skew detection (in progress) @@ -230,4 +230,4 @@ Thanks goes to these incredible people: - \ No newline at end of file + From 146e36df5809f76d00900d0ee787bf9aa3f078bd Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Jul 2022 10:02:24 -0700 Subject: [PATCH 17/26] fix: Deprecate 3.7 wheels and fix verification workflow (#2934) * Remove 3.7 wheels Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 1b0059f1a31..30b40df3b35 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -187,6 +187,18 @@ jobs: cd dist/ pip install wheel for f in *.whl; do pip install $f || true; done + - name: Install apache-arrow on ubuntu + if: ${{ matrix.from-source && matrix.os == 'ubuntu-latest' }} + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: ${{ matrix.from-source && matrix.os == 'macos-10.15' && matrix.python-version != '3.10' }} + run: brew install apache-arrow - name: Install dist with go if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}} env: From a71b9d09a249c908716d9f33a626a6a69f45761e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:36:07 -0700 Subject: [PATCH 18/26] chore(deps): Bump moment from 2.29.3 to 2.29.4 in /sdk/python/feast/ui (#2925) chore(deps): bump moment from 2.29.3 to 2.29.4 in /sdk/python/feast/ui Bumps [moment](https://github.com/moment/moment) from 2.29.3 to 2.29.4. - [Release notes](https://github.com/moment/moment/releases) - [Changelog](https://github.com/moment/moment/blob/develop/CHANGELOG.md) - [Commits](https://github.com/moment/moment/compare/2.29.3...2.29.4) --- updated-dependencies: - dependency-name: moment dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index ec046557236..883c19660b2 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -13,7 +13,7 @@ "@types/d3": "^7.1.0", "d3": "^7.4.4", "inter-ui": "^3.19.3", - "moment": "^2.29.3", + "moment": "^2.29.4", "prop-types": "^15.8.1", "query-string": "^7.1.1", "react": "^18.1.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 83157394caf..f2fd12b4e57 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -7143,10 +7143,10 @@ mkdirp@~0.5.1: dependencies: minimist "^1.2.6" -moment@^2.29.1, moment@^2.29.3: - version "2.29.3" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" - integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== +moment@^2.29.1, moment@^2.29.4: + version "2.29.4" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== ms@2.0.0: version "2.0.0" From 922dcd3be3c359be64addbd306a29a7424414588 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Thu, 14 Jul 2022 18:32:22 -0700 Subject: [PATCH 19/26] chore: Fixes and Readme for python<>go interface (#2936) * chore: Fixes and Readme for python<>go interface Signed-off-by: Achal Shah * cr Signed-off-by: Achal Shah * Switch to cgo allocator Signed-off-by: Felix Wang * Fix memory leak Signed-off-by: Felix Wang * Fix another memory leak Signed-off-by: Felix Wang * Do not use cgo allocator for memory buffers Signed-off-by: Felix Wang * Switch to cgo allocator for memory buffers Signed-off-by: Felix Wang * Switch test to pass Signed-off-by: Felix Wang * Use more idiomatic way to test truth Signed-off-by: Felix Wang * Update docs Signed-off-by: Felix Wang Co-authored-by: Felix Wang --- go/README.md | 109 ++++++++++++++++++ go/embedded/online_features.go | 37 +++++- go/internal/feast/featurestore.go | 2 +- go/internal/feast/onlineserving/serving.go | 10 ++ .../feast/server/logging/memorybuffer.go | 2 +- .../feast/server/logging/memorybuffer_test.go | 4 +- .../embedded_go/online_features_service.py | 1 + 7 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 go/README.md diff --git a/go/README.md b/go/README.md new file mode 100644 index 00000000000..0bca470919f --- /dev/null +++ b/go/README.md @@ -0,0 +1,109 @@ +This directory contains the Go logic that's executed by the `EmbeddedOnlineFeatureServer` from Python. + +## Building and Linking +[gopy](https://github.com/go-python/gopy) generates (and compiles) a CPython extension module from a Go package. That's what we're using here, as visible in [setup.py](../setup.py). + +Under the hood, gopy invokes `go build`, and then templates `cgo` stubs for the Go module that exposes the public functions from the Go module as C functions. +For our project, this stuff can be found at `sdk/python/feast/embedded_go/lib/embedded.go` & `sdk/python/feast/embedded_go/lib/embedded_go.h` after running `make compile-go-lib`. + +## Arrow memory management +Understanding this is the trickiest part of this integration. + +At a high level, when using the Python<>Go integration, the Python layer exports request data into an [Arrow Record batch](https://arrow.apache.org/docs/python/data.html) which is transferred to Go using Arrow's zero copy mechanism. +Similarly, the Go layer converts feature values read from the online store into a Record Batch that's exported to Python using the same mechanics. + +The first thing to note is that from the Python perspective, all the export logic assumes that we're exporting to & importing from C, not Go. This is because pyarrow only interops with C, and the fact we're using Go is an implementation detail not relevant to the Python layer. + +### Export Entities & Request data from Python to Go +The code exporting to C is this, in [online_feature_service.py](../sdk/python/feast/embedded_go/online_features_service.py) +``` +( + entities_c_schema, + entities_ptr_schema, + entities_c_array, + entities_ptr_array, +) = allocate_schema_and_array() +( + req_data_c_schema, + req_data_ptr_schema, + req_data_c_array, + req_data_ptr_array, +) = allocate_schema_and_array() + +batch, schema = map_to_record_batch(entities, join_keys_types) +schema._export_to_c(entities_ptr_schema) +batch._export_to_c(entities_ptr_array) + +batch, schema = map_to_record_batch(request_data) +schema._export_to_c(req_data_ptr_schema) +batch._export_to_c(req_data_ptr_array) +``` + +Under the hood, `allocate_schema_and_array` allocates a pointer (`struct ArrowSchema*` and `struct ArrowArray*`) in native memory (i.e. the C layer) using `cffi`. +Next, the RecordBatch exports to this pointer using [`_export_to_c`](https://github.com/apache/arrow/blob/master/python/pyarrow/table.pxi#L2509), which uses [`ExportRecordBatch`](https://arrow.apache.org/docs/cpp/api/c_abi.html#_CPPv417ExportRecordBatchRK11RecordBatchP10ArrowArrayP11ArrowSchema) under the hood. + +As per the documentation for ExportRecordBatch: +> Status ExportRecordBatch(const RecordBatch &batch, struct ArrowArray *out, struct ArrowSchema *out_schema = NULLPTR) +> Export C++ RecordBatch using the C data interface format. +> +> The record batch is exported as if it were a struct array. The resulting ArrowArray struct keeps the record batch data and buffers alive until its release callback is called by the consumer. + +This is why `GetOnlineFeatures()` in `online_features.go` calls `record.Release()` as below: +``` +entitiesRecord, err := readArrowRecord(entities) +if err != nil { + return err +} +defer entitiesRecord.Release() +... +requestDataRecords, err := readArrowRecord(requestData) +if err != nil { + return err +} +defer requestDataRecords.Release() +``` + +Additionally, we need to pass in a pair of pointers to `GetOnlineFeatures()` that are populated by the Go layer, and the resultant feature values can be passed back to Python (via the C layer) using zero-copy semantics. +That happens as follows: +``` +( + features_c_schema, + features_ptr_schema, + features_c_array, + features_ptr_array, +) = allocate_schema_and_array() + +... + +record_batch = pa.RecordBatch._import_from_c( + features_ptr_array, features_ptr_schema +) +``` + +The corresponding Go code that exports this data is: +``` +result := array.NewRecord(arrow.NewSchema(outputFields, nil), outputColumns, int64(numRows)) + +cdata.ExportArrowRecordBatch(result, + cdata.ArrayFromPtr(output.DataPtr), + cdata.SchemaFromPtr(output.SchemaPtr)) +``` + +The documentation for `ExportArrowRecordBatch` is great. It has this super useful caveat: + +> // The release function on the populated CArrowArray will properly decrease the reference counts, +> // and release the memory if the record has already been released. But since this must be explicitly +> // done, make sure it is released so that you do not create a memory leak. + +This implies that the reciever is on the hook for explicitly releasing this memory. + +However, we're using `_import_from_c`, which uses [`ImportRecordBatch`](https://arrow.apache.org/docs/cpp/api/c_abi.html#_CPPv417ImportRecordBatchP10ArrowArrayP11ArrowSchema), which implies that the receiver of the RecordBatch is the new owner of the data. +This is wrapped by pyarrow - and when the corresponding python object goes out of scope, it should clean up the underlying record batch. + +Another thing to note (which I'm not sure may be the source of issues) is that Arrow has the concept of [Memory Pools](https://arrow.apache.org/docs/python/api/memory.html#memory-pools). +Memory pools can be set in python as well as in Go. I *believe* that if we use the CGoArrowAllocator, that uses whatever pool C++ uses, which should be the same as the one used by PyArrow. But this should be vetted. + + +### References +- https://arrow.apache.org/docs/format/CDataInterface.html#memory-management +- https://arrow.apache.org/docs/python/memory.html \ No newline at end of file diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index e5860507e45..7fd34d16e40 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -33,6 +33,11 @@ type OnlineFeatureService struct { grpcStopCh chan os.Signal httpStopCh chan os.Signal + statusColumnBuildersToRelease []*array.Int32Builder + tsColumnBuildersToRelease []*array.Int64Builder + arraysToRelease []arrow.Array + resultsToRelease []arrow.Record + err error } @@ -143,6 +148,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures( if err != nil { return err } + defer entitiesRecord.Release() numRows := entitiesRecord.Column(0).Len() @@ -155,6 +161,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures( if err != nil { return err } + defer requestDataRecords.Release() requestDataProto, err := recordToProto(requestDataRecords) if err != nil { @@ -178,6 +185,24 @@ func (s *OnlineFeatureService) GetOnlineFeatures( return err } + // Release all objects that are no longer required. + for _, statusColumnBuilderToRelease := range s.statusColumnBuildersToRelease { + statusColumnBuilderToRelease.Release() + } + for _, tsColumnBuilderToRelease := range s.tsColumnBuildersToRelease { + tsColumnBuilderToRelease.Release() + } + for _, arrayToRelease := range s.arraysToRelease { + arrayToRelease.Release() + } + for _, resultsToRelease := range s.resultsToRelease { + resultsToRelease.Release() + } + s.statusColumnBuildersToRelease = nil + s.tsColumnBuildersToRelease = nil + s.arraysToRelease = nil + s.resultsToRelease = nil + outputFields := make([]arrow.Field, 0) outputColumns := make([]arrow.Array, 0) pool := memory.NewCgoArrowAllocator() @@ -210,13 +235,19 @@ func (s *OnlineFeatureService) GetOnlineFeatures( } tsColumn := tsColumnBuilder.NewArray() outputColumns = append(outputColumns, tsColumn) + + // Mark builders and arrays for release. + s.statusColumnBuildersToRelease = append(s.statusColumnBuildersToRelease, statusColumnBuilder) + s.tsColumnBuildersToRelease = append(s.tsColumnBuildersToRelease, tsColumnBuilder) + s.arraysToRelease = append(s.arraysToRelease, statusColumn) + s.arraysToRelease = append(s.arraysToRelease, tsColumn) + s.arraysToRelease = append(s.arraysToRelease, featureVector.Values) } result := array.NewRecord(arrow.NewSchema(outputFields, nil), outputColumns, int64(numRows)) + s.resultsToRelease = append(s.resultsToRelease, result) - cdata.ExportArrowRecordBatch(result, - cdata.ArrayFromPtr(output.DataPtr), - cdata.SchemaFromPtr(output.SchemaPtr)) + cdata.ExportArrowRecordBatch(result, cdata.ArrayFromPtr(output.DataPtr), cdata.SchemaFromPtr(output.SchemaPtr)) return nil } diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index ad1f94a4ba2..ed38411460a 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -113,7 +113,7 @@ func (fs *FeatureStore) GetOnlineFeatures( } result := make([]*onlineserving.FeatureVector, 0) - arrowMemory := memory.NewGoAllocator() + arrowMemory := memory.NewCgoArrowAllocator() featureViews := make([]*model.FeatureView, len(requestedFeatureViews)) index := 0 for _, featuresAndView := range requestedFeatureViews { diff --git a/go/internal/feast/onlineserving/serving.go b/go/internal/feast/onlineserving/serving.go index e2a2df923be..3c6f5451537 100644 --- a/go/internal/feast/onlineserving/serving.go +++ b/go/internal/feast/onlineserving/serving.go @@ -415,6 +415,8 @@ func KeepOnlyRequestedFeatures( vectorsByName := make(map[string]*FeatureVector) expectedVectors := make([]*FeatureVector, 0) + usedVectors := make(map[string]bool) + for _, vector := range vectors { vectorsByName[vector.Name] = vector } @@ -438,6 +440,14 @@ func KeepOnlyRequestedFeatures( return nil, fmt.Errorf("requested feature %s can't be retrieved", featureRef) } expectedVectors = append(expectedVectors, vectorsByName[qualifiedName]) + usedVectors[qualifiedName] = true + } + + // Free arrow arrays for vectors that were not used. + for _, vector := range vectors { + if _, ok := usedVectors[vector.Name]; !ok { + vector.Values.Release() + } } return expectedVectors, nil diff --git a/go/internal/feast/server/logging/memorybuffer.go b/go/internal/feast/server/logging/memorybuffer.go index 9ffb0ff73b8..c9f00218dfc 100644 --- a/go/internal/feast/server/logging/memorybuffer.go +++ b/go/internal/feast/server/logging/memorybuffer.go @@ -128,7 +128,7 @@ func getArrowSchema(schema *FeatureServiceSchema) (*arrow.Schema, error) { // and writes them to arrow table. // Returns arrow table that contains all of the logs in columnar format. func (b *MemoryBuffer) convertToArrowRecord() (arrow.Record, error) { - arrowMemory := memory.NewGoAllocator() + arrowMemory := memory.NewCgoArrowAllocator() numRows := len(b.logs) columns := make(map[string][]*types.Value) diff --git a/go/internal/feast/server/logging/memorybuffer_test.go b/go/internal/feast/server/logging/memorybuffer_test.go index 94f0f86ef02..ec83680f4ff 100644 --- a/go/internal/feast/server/logging/memorybuffer_test.go +++ b/go/internal/feast/server/logging/memorybuffer_test.go @@ -118,7 +118,7 @@ func TestSerializeToArrowTable(t *testing.T) { LogTimestamp: time.Now(), }) - pool := memory.NewGoAllocator() + pool := memory.NewCgoArrowAllocator() builder := array.NewRecordBuilder(pool, b.arrowSchema) defer builder.Release() @@ -159,7 +159,7 @@ func TestSerializeToArrowTable(t *testing.T) { expectedRecord := builder.NewRecord() assert.Nil(t, err) for colIdx := 0; colIdx < int(record.NumCols()); colIdx++ { - assert.Equal(t, expectedRecord.Column(colIdx), record.Column(colIdx), "Columns with idx %d are not equal", colIdx) + assert.True(t, array.Equal(expectedRecord.Column(colIdx), record.Column(colIdx)), "Columns with idx %d are not equal", colIdx) } } diff --git a/sdk/python/feast/embedded_go/online_features_service.py b/sdk/python/feast/embedded_go/online_features_service.py index 3081843778c..d9b34b2414c 100644 --- a/sdk/python/feast/embedded_go/online_features_service.py +++ b/sdk/python/feast/embedded_go/online_features_service.py @@ -147,6 +147,7 @@ def get_online_features( features_ptr_array, features_ptr_schema ) resp = record_batch_to_online_response(record_batch) + del record_batch return OnlineResponse(resp) def start_grpc_server( From 9b9fbbe692475d85b713b7e6707a105fca35fbf9 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 23:32:02 -0700 Subject: [PATCH 20/26] fix: Update gopy to point to fork to resolve github annotation errors. (#2940) * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- go.mod | 2 ++ go.sum | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ef64636e73e..3c05383ffc2 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/feast-dev/feast go 1.17 +replace github.com/go-python/gopy v0.4.4 => github.com/feast-dev/gopy v0.4.1-0.20220714211711-252048177d85 + require ( github.com/apache/arrow/go/v8 v8.0.0 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index d0b7f8fcd80..11f1ba4d74e 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,14 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/feast-dev/gopy v0.4.1-0.20220714205859-591500e3215f h1:tTjEpVu4H/ZGh4wo3WETbA9dutNM6bXMXvyZbb9GLCs= +github.com/feast-dev/gopy v0.4.1-0.20220714205859-591500e3215f/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= +github.com/feast-dev/gopy v0.4.1-0.20220714211038-aa312c13fd79 h1:oFj6GDGR8E4S5GeMyLBvaKtvMZxj3hHqsB5Xndjxjz8= +github.com/feast-dev/gopy v0.4.1-0.20220714211038-aa312c13fd79/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= +github.com/feast-dev/gopy v0.4.1-0.20220714211330-67b016d61ed4 h1:UfzPdqqAfrt8f+jDIY61lbzqFZYsX2BhVyNcCbdpE+U= +github.com/feast-dev/gopy v0.4.1-0.20220714211330-67b016d61ed4/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= +github.com/feast-dev/gopy v0.4.1-0.20220714211711-252048177d85 h1:BKmfqWiDbxvviB6vemPbbNjF+ywRsBMCdk1QvrcGgkc= +github.com/feast-dev/gopy v0.4.1-0.20220714211711-252048177d85/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -110,8 +118,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-python/gopy v0.4.4 h1:3LTsrfVcmg2VEM6wU+eh4d9EZn5H2iogObXjiQHrF8Q= -github.com/go-python/gopy v0.4.4/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= From c8079c126f685ab277e013a4173700e802c34672 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Fri, 15 Jul 2022 13:18:09 -0700 Subject: [PATCH 21/26] chore: More automated upgrades in repo definitions (#2941) Signed-off-by: Achal Shah --- sdk/python/feast/repo_upgrade.py | 103 +++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/sdk/python/feast/repo_upgrade.py b/sdk/python/feast/repo_upgrade.py index 5c8d7433b2d..6aa7a2cc1d4 100644 --- a/sdk/python/feast/repo_upgrade.py +++ b/sdk/python/feast/repo_upgrade.py @@ -3,6 +3,7 @@ from typing import Dict, List from bowler import Query +from fissix.fixer_util import touch_import from fissix.pgen2 import token from fissix.pygram import python_symbols from fissix.pytree import Node @@ -30,6 +31,39 @@ def __init__(self, repo_path: str, write: bool): def upgrade(self): self.remove_date_partition_column() + self.rename_features_to_schema() + + def rename_inputs_to_sources(self): + def _change_argument_transform(node, capture, filename) -> None: + children = node.children + self.rename_arguments_in_children(children, {"inputs": "sources"}) + + PATTERN = """ + decorator< + any * + "on_demand_feature_view" + any * + > + """ + + Query(self.repo_files).select(PATTERN).modify( + _change_argument_transform + ).execute(write=self.write, interactive=False) + + def rename_features_to_schema(self): + Query(str(self.repo_path)).select_class("Feature").modify( + self.import_remover("Feature") + ).execute(interactive=False, write=self.write) + + def _rename_class_name( + node: Node, capture: Dict[str, Node], filename: str + ) -> None: + self.rename_class_call(node, "Field") + touch_import("feast", "Field", node) + + Query(self.repo_files).select_class("Feature").is_call().modify( + _rename_class_name + ).execute(write=self.write, interactive=False) def remove_date_partition_column(self): def _remove_date_partition_column( @@ -42,6 +76,52 @@ def _remove_date_partition_column( _remove_date_partition_column ).execute(write=self.write, interactive=False) + @staticmethod + def rename_arguments_in_children( + children: List[Node], renames: Dict[str, str] + ) -> None: + """ + Renames the arguments in the children list of a node by searching for the + argument list or trailing list and renaming all keys in `renames` dict to + corresponding value. + """ + for child in children: + if not isinstance(child, Node): + continue + if ( + child.type == python_symbols.arglist + or child.type == python_symbols.trailer + ): + if not child.children: + continue + for _, child in enumerate(child.children): + if not isinstance(child, Node): + continue + else: + if child.type == python_symbols.argument: + if child.children[0].value in renames: + child.children[0].value = renames[ + child.children[0].value + ] + + @staticmethod + def rename_class_call(node: Node, new_class_name: str): + """ + Rename the class being instantiated. + f = Feature( + name="driver_id", + join_key="driver_id", + ) + into + f = Field( + name="driver_id", + ) + This method assumes that node represents a class call that already has an arglist. + """ + if len(node.children) < 2 or len(node.children[1].children) < 2: + raise ValueError(f"Expected a class call with an arglist but got {node}.") + node.children[0].value = new_class_name + @staticmethod def remove_argument_transform(node: Node, argument: str): """ @@ -70,3 +150,26 @@ def remove_argument_transform(node: Node, argument: str): class_args.pop(i) if i < len(class_args) and class_args[i].type == token.NEWLINE: class_args.pop(i) + + @staticmethod + def import_remover(class_name): + def remove_import_transformer(node, capture, filename): + if "class_import" in capture and capture["class_name"].value == class_name: + if capture["class_import"].type == python_symbols.import_from: + import_from_stmt = node.children + imported_classes = import_from_stmt[3] + + if len(imported_classes.children) > 1: + # something of the form `from feast import A, ValueType` + for i, class_leaf in enumerate(imported_classes.children): + if class_leaf.value == class_name: + imported_classes.children.pop(i) + if i == len(imported_classes.children): + imported_classes.children.pop(i - 1) + else: + imported_classes.children.pop(i) + else: + # something of the form `from feast import ValueType` + node.parent.children.remove(node) + + return remove_import_transformer From f341703bb8f07ddb2cf1ced83c0294e51814b38c Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Mon, 18 Jul 2022 16:11:38 -0400 Subject: [PATCH 22/26] chore: Add project metadata to registry (#2938) * chore: Add project metadata to registry Signed-off-by: Danny Chiao * lint Signed-off-by: Danny Chiao * add to file registry Signed-off-by: Danny Chiao * add to sql registry Signed-off-by: Danny Chiao * fixes Signed-off-by: Danny Chiao * fixes Signed-off-by: Danny Chiao * fixes Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * lint Signed-off-by: Danny Chiao * fix test Signed-off-by: Danny Chiao * add tests Signed-off-by: Danny Chiao * make backwards compatible Signed-off-by: Danny Chiao * update makefile for passing local tests Signed-off-by: Danny Chiao * update makefile for passing local tests Signed-off-by: Danny Chiao * update makefile for passing local tests Signed-off-by: Danny Chiao * update makefile for passing local tests Signed-off-by: Danny Chiao --- .../workflows/pr_local_integration_tests.yml | 17 ++ Makefile | 3 +- protos/feast/core/Registry.proto | 8 +- .../source/feast.infra.materialization.rst | 29 +++ sdk/python/docs/source/feast.infra.rst | 1 + sdk/python/docs/source/feast.rst | 16 ++ sdk/python/feast/driver_test_data.py | 2 +- sdk/python/feast/feature_store.py | 6 +- sdk/python/feast/infra/registry_stores/sql.py | 73 ++++++- sdk/python/feast/project_metadata.py | 111 ++++++++++ sdk/python/feast/registry.py | 192 ++++++++++++++---- sdk/python/feast/templates/gcp/driver_repo.py | 1 + sdk/python/feast/templates/hbase/example.py | 1 + sdk/python/feast/templates/local/example.py | 1 + sdk/python/feast/usage.py | 5 + .../feature_repos/repo_configuration.py | 2 +- .../integration/registration/test_registry.py | 36 +++- .../registration/test_sql_registry.py | 14 ++ 18 files changed, 450 insertions(+), 68 deletions(-) create mode 100644 sdk/python/docs/source/feast.infra.materialization.rst create mode 100644 sdk/python/feast/project_metadata.py diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 08470833936..4c87780888c 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -67,6 +67,23 @@ jobs: sudo apt install -y -V libarrow-dev - name: Install dependencies run: make install-python-ci-dependencies + - name: Set up gcloud SDK # TODO(adchia): remove this dependency + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak run: make test-python-integration-local + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: localintegrationtests + env_vars: OS,PYTHON + fail_ci_if_error: true + verbose: true diff --git a/Makefile b/Makefile index 6d733ac61f9..915ac907f7c 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ test-python-integration-local: python -m pytest -n 8 --integration \ -k "not test_apply_entity_integration and \ not test_apply_feature_view_integration and \ - not test_apply_data_source_integration" \ + not test_apply_data_source_integration and \ + not test_lambda_materialization" \ sdk/python/tests \ ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; diff --git a/protos/feast/core/Registry.proto b/protos/feast/core/Registry.proto index 19f17a81589..7d80d8c837f 100644 --- a/protos/feast/core/Registry.proto +++ b/protos/feast/core/Registry.proto @@ -34,7 +34,7 @@ import "feast/core/SavedDataset.proto"; import "feast/core/ValidationProfile.proto"; import "google/protobuf/timestamp.proto"; -// Next id: 15 +// Next id: 16 message Registry { repeated Entity entities = 1; repeated FeatureTable feature_tables = 2; @@ -47,9 +47,15 @@ message Registry { repeated SavedDataset saved_datasets = 11; repeated ValidationReference validation_references = 13; Infra infra = 10; + // Tracking metadata of Feast by project + repeated ProjectMetadata project_metadata = 15; string registry_schema_version = 3; // to support migrations; incremented when schema is changed string version_id = 4; // version id, random string generated on each update of the data; now used only for debugging purposes google.protobuf.Timestamp last_updated = 5; +} +message ProjectMetadata { + string project = 1; + string project_uuid = 2; } diff --git a/sdk/python/docs/source/feast.infra.materialization.rst b/sdk/python/docs/source/feast.infra.materialization.rst new file mode 100644 index 00000000000..49fdc404cb7 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.rst @@ -0,0 +1,29 @@ +feast.infra.materialization package +=================================== + +Submodules +---------- + +feast.infra.materialization.batch\_materialization\_engine module +----------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.batch_materialization_engine + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.local\_engine module +------------------------------------------------ + +.. automodule:: feast.infra.materialization.local_engine + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.materialization + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index ec2cc120a63..42c7d1334b6 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -7,6 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 + feast.infra.materialization feast.infra.offline_stores feast.infra.online_stores feast.infra.registry_stores diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst index 7c569fc7130..c000ac2e2b7 100644 --- a/sdk/python/docs/source/feast.rst +++ b/sdk/python/docs/source/feast.rst @@ -225,6 +225,14 @@ feast.online\_response module :undoc-members: :show-inheritance: +feast.project\_metadata module +------------------------------ + +.. automodule:: feast.project_metadata + :members: + :undoc-members: + :show-inheritance: + feast.proto\_json module ------------------------ @@ -273,6 +281,14 @@ feast.repo\_operations module :undoc-members: :show-inheritance: +feast.repo\_upgrade module +-------------------------- + +.. automodule:: feast.repo_upgrade + :members: + :undoc-members: + :show-inheritance: + feast.request\_feature\_view module ----------------------------------- diff --git a/sdk/python/feast/driver_test_data.py b/sdk/python/feast/driver_test_data.py index 117bfcbd9cb..991b5391e85 100644 --- a/sdk/python/feast/driver_test_data.py +++ b/sdk/python/feast/driver_test_data.py @@ -98,7 +98,7 @@ def create_driver_hourly_stats_df(drivers, start_date, end_date) -> pd.DataFrame "event_timestamp": [ pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") for dt in pd.date_range( - start=start_date, end=end_date, freq="1H", closed="left" + start=start_date, end=end_date, freq="1H", inclusive="left" ) ] # include a fixed timestamp for get_historical_features in the quickstart diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index ce72c3c8034..c49ce7dc77e 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -142,7 +142,7 @@ def __init__( self._registry = SqlRegistry(registry_config, None) else: r = Registry(registry_config, repo_path=self.repo_path) - r._initialize_registry() + r._initialize_registry(self.config.project) self._registry = r self._provider = get_provider(self.config, self.repo_path) self._go_server = None @@ -183,7 +183,7 @@ def refresh_registry(self): """ registry_config = self.config.get_registry_config() registry = Registry(registry_config, repo_path=self.repo_path) - registry.refresh() + registry.refresh(self.config.project) self._registry = registry @@ -704,7 +704,7 @@ def plan( # Compute the desired difference between the current infra, as stored in the registry, # and the desired infra. - self._registry.refresh() + self._registry.refresh(self.project) current_infra_proto = self._registry.proto().infra.__deepcopy__() desired_registry_proto = desired_repo_contents.to_registry_proto() new_infra = self._provider.plan_infra(self.config, desired_registry_proto) diff --git a/sdk/python/feast/infra/registry_stores/sql.py b/sdk/python/feast/infra/registry_stores/sql.py index 7ea4a96849a..2d3ac9d6831 100644 --- a/sdk/python/feast/infra/registry_stores/sql.py +++ b/sdk/python/feast/infra/registry_stores/sql.py @@ -1,4 +1,6 @@ +import uuid from datetime import datetime +from enum import Enum from pathlib import Path from typing import Any, List, Optional, Set, Union @@ -17,6 +19,7 @@ ) from sqlalchemy.engine import Engine +from feast import usage from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -32,6 +35,7 @@ from feast.feature_view import FeatureView from feast.infra.infra_object import Infra from feast.on_demand_feature_view import OnDemandFeatureView +from feast.project_metadata import ProjectMetadata from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.protos.feast.core.Entity_pb2 import Entity as EntityProto from feast.protos.feast.core.FeatureService_pb2 import ( @@ -156,6 +160,12 @@ Column("infra_proto", LargeBinary, nullable=False), ) + +class FeastMetadataKeys(Enum): + LAST_UPDATED_TIMESTAMP = "last_updated_timestamp" + PROJECT_UUID = "project_uuid" + + feast_metadata = Table( "feast_metadata", metadata, @@ -189,7 +199,7 @@ def teardown(self): stmt = delete(t) conn.execute(stmt) - def refresh(self): + def refresh(self, project: Optional[str]): # This method is a no-op since we're always reading the latest values from the db. pass @@ -459,6 +469,22 @@ def list_on_demand_feature_views( "feature_view_proto", ) + def list_project_metadata( + self, project: str, allow_cache: bool = False + ) -> List[ProjectMetadata]: + with self.engine.connect() as conn: + stmt = select(feast_metadata).where(feast_metadata.c.project_id == project,) + rows = conn.execute(stmt).all() + if rows: + project_metadata = ProjectMetadata(project_name=project) + for row in rows: + if row["metadata_key"] == FeastMetadataKeys.PROJECT_UUID.value: + project_metadata.project_uuid = row["metadata_value"] + break + # TODO(adchia): Add other project metadata in a structured way + return [project_metadata] + return [] + def apply_saved_dataset( self, saved_dataset: SavedDataset, project: str, commit: bool = True, ): @@ -629,6 +655,7 @@ def proto(self) -> RegistryProto: (self.list_feature_services, r.feature_services), (self.list_saved_datasets, r.saved_datasets), (self.list_validation_references, r.validation_references), + (self.list_project_metadata, r.project_metadata), ]: objs: List[Any] = lister(project) # type: ignore if objs: @@ -651,14 +678,16 @@ def commit(self): def _apply_object( self, table, project: str, id_field_name, obj, proto_field_name, name=None ): + self._maybe_init_project_metadata(project) + name = name or obj.name with self.engine.connect() as conn: + update_datetime = datetime.utcnow() + update_time = int(update_datetime.timestamp()) stmt = select(table).where( getattr(table.c, id_field_name) == name, table.c.project_id == project ) row = conn.execute(stmt).first() - update_datetime = datetime.utcnow() - update_time = int(update_datetime.timestamp()) if hasattr(obj, "last_updated_timestamp"): obj.last_updated_timestamp = update_datetime @@ -685,6 +714,30 @@ def _apply_object( self._set_last_updated_metadata(update_datetime, project) + def _maybe_init_project_metadata(self, project): + # Initialize project metadata if needed + with self.engine.connect() as conn: + update_datetime = datetime.utcnow() + update_time = int(update_datetime.timestamp()) + stmt = select(feast_metadata).where( + feast_metadata.c.metadata_key == FeastMetadataKeys.PROJECT_UUID.value, + feast_metadata.c.project_id == project, + ) + row = conn.execute(stmt).first() + if row: + usage.set_current_project_uuid(row["metadata_value"]) + else: + new_project_uuid = f"{uuid.uuid4()}" + values = { + "metadata_key": FeastMetadataKeys.PROJECT_UUID.value, + "metadata_value": new_project_uuid, + "last_updated_timestamp": update_time, + "project_id": project, + } + insert_stmt = insert(feast_metadata).values(values) + conn.execute(insert_stmt) + usage.set_current_project_uuid(new_project_uuid) + def _delete_object(self, table, name, project, id_field_name, not_found_exception): with self.engine.connect() as conn: stmt = delete(table).where( @@ -708,6 +761,8 @@ def _get_object( proto_field_name, not_found_exception, ): + self._maybe_init_project_metadata(project) + with self.engine.connect() as conn: stmt = select(table).where( getattr(table.c, id_field_name) == name, table.c.project_id == project @@ -721,6 +776,7 @@ def _get_object( def _list_objects( self, table, project, proto_class, python_class, proto_field_name ): + self._maybe_init_project_metadata(project) with self.engine.connect() as conn: stmt = select(table).where(table.c.project_id == project) rows = conn.execute(stmt).all() @@ -736,7 +792,8 @@ def _list_objects( def _set_last_updated_metadata(self, last_updated: datetime, project: str): with self.engine.connect() as conn: stmt = select(feast_metadata).where( - feast_metadata.c.metadata_key == "last_updated_timestamp", + feast_metadata.c.metadata_key + == FeastMetadataKeys.LAST_UPDATED_TIMESTAMP.value, feast_metadata.c.project_id == project, ) row = conn.execute(stmt).first() @@ -744,7 +801,7 @@ def _set_last_updated_metadata(self, last_updated: datetime, project: str): update_time = int(last_updated.timestamp()) values = { - "metadata_key": "last_updated_timestamp", + "metadata_key": FeastMetadataKeys.LAST_UPDATED_TIMESTAMP.value, "metadata_value": f"{update_time}", "last_updated_timestamp": update_time, "project_id": project, @@ -753,7 +810,8 @@ def _set_last_updated_metadata(self, last_updated: datetime, project: str): update_stmt = ( update(feast_metadata) .where( - feast_metadata.c.metadata_key == "last_updated_timestamp", + feast_metadata.c.metadata_key + == FeastMetadataKeys.LAST_UPDATED_TIMESTAMP.value, feast_metadata.c.project_id == project, ) .values(values) @@ -766,7 +824,8 @@ def _set_last_updated_metadata(self, last_updated: datetime, project: str): def _get_last_updated_metadata(self, project: str): with self.engine.connect() as conn: stmt = select(feast_metadata).where( - feast_metadata.c.metadata_key == "last_updated_timestamp", + feast_metadata.c.metadata_key + == FeastMetadataKeys.LAST_UPDATED_TIMESTAMP.value, feast_metadata.c.project_id == project, ) row = conn.execute(stmt).first() diff --git a/sdk/python/feast/project_metadata.py b/sdk/python/feast/project_metadata.py new file mode 100644 index 00000000000..829e9ff0d54 --- /dev/null +++ b/sdk/python/feast/project_metadata.py @@ -0,0 +1,111 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid +from typing import Optional + +from google.protobuf.json_format import MessageToJson +from typeguard import typechecked + +from feast.protos.feast.core.Registry_pb2 import ProjectMetadata as ProjectMetadataProto +from feast.usage import log_exceptions + + +@typechecked +class ProjectMetadata: + """ + Tracks project level metadata + + Attributes: + project_name: The registry-scoped unique name of the project. + project_uuid: The UUID for this project + """ + + project_name: str + project_uuid: str + + @log_exceptions + def __init__( + self, + *args, + project_name: Optional[str] = None, + project_uuid: Optional[str] = None, + ): + """ + Creates an Project metadata object. + + Args: + project_name: The registry-scoped unique name of the project. + project_uuid: The UUID for this project + + Raises: + ValueError: Parameters are specified incorrectly. + """ + if not project_name: + raise ValueError("Project name needs to be specified") + + self.project_name = project_name + self.project_uuid = project_uuid or f"{uuid.uuid4()}" + + def __hash__(self) -> int: + return hash((self.project_name, self.project_uuid)) + + def __eq__(self, other): + if not isinstance(other, ProjectMetadata): + raise TypeError( + "Comparisons should only involve ProjectMetadata class objects." + ) + + if ( + self.project_name != other.project_name + or self.project_uuid != other.project_uuid + ): + return False + + return True + + def __str__(self): + return str(MessageToJson(self.to_proto())) + + def __lt__(self, other): + return self.project_name < other.project_name + + @classmethod + def from_proto(cls, project_metadata_proto: ProjectMetadataProto): + """ + Creates project metadata from a protobuf representation. + + Args: + project_metadata_proto: A protobuf representation of project metadata. + + Returns: + A ProjectMetadata object based on the protobuf. + """ + entity = cls( + project_name=project_metadata_proto.project, + project_uuid=project_metadata_proto.project_uuid, + ) + + return entity + + def to_proto(self) -> ProjectMetadataProto: + """ + Converts a project metadata object to its protobuf representation. + + Returns: + An ProjectMetadataProto protobuf. + """ + + return ProjectMetadataProto( + project=self.project_name, project_uuid=self.project_uuid + ) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index c721bd648a8..f72fd717d23 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -14,6 +14,7 @@ import abc import json import logging +import uuid from abc import abstractmethod from collections import defaultdict from datetime import datetime, timedelta @@ -28,6 +29,7 @@ from google.protobuf.json_format import MessageToJson from proto import Message +from feast import usage from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -47,6 +49,8 @@ from feast.importer import import_class from feast.infra.infra_object import Infra from feast.on_demand_feature_view import OnDemandFeatureView +from feast.project_metadata import ProjectMetadata +from feast.protos.feast.core.Registry_pb2 import ProjectMetadata as ProjectMetadataProto from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.registry_store import NoopRegistryStore from feast.repo_config import RegistryConfig @@ -57,7 +61,6 @@ REGISTRY_SCHEMA_VERSION = "1" - REGISTRY_STORE_CLASS_FOR_TYPE = { "GCSRegistryStore": "feast.infra.gcp.GCSRegistryStore", "S3RegistryStore": "feast.infra.aws.S3RegistryStore", @@ -121,7 +124,6 @@ def get_objects_from_repo_contents( FEAST_OBJECT_TYPES = [feast_object_type for feast_object_type in FeastObjectType] - logger = logging.getLogger(__name__) @@ -607,6 +609,20 @@ def list_validation_references( List of request feature views """ + def list_project_metadata( + self, project: str, allow_cache: bool = False + ) -> List[ProjectMetadata]: + """ + Retrieves project metadata + + Args: + project: Filter metadata based on project name + allow_cache: Allow returning feature views from the cached registry + + Returns: + List of project metadata + """ + @abstractmethod def update_infra(self, infra: Infra, project: str, commit: bool = True): """ @@ -660,7 +676,7 @@ def commit(self): """Commits the state of the registry cache to the remote registry store.""" @abstractmethod - def refresh(self): + def refresh(self, project: Optional[str]): """Refreshes the state of the registry cache by fetching the registry state from the remote registry store.""" @staticmethod @@ -678,6 +694,10 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: """ registry_dict: Dict[str, Any] = defaultdict(list) registry_dict["project"] = project + for project_metadata in sorted(self.list_project_metadata(project=project)): + registry_dict["projectMetadata"].append( + self._message_to_sorted_dict(project_metadata.to_proto()) + ) for data_source in sorted( self.list_data_sources(project=project), key=lambda ds: ds.name ): @@ -733,6 +753,25 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: return registry_dict +def _get_project_metadata( + registry_proto: Optional[RegistryProto], project: str +) -> Optional[ProjectMetadataProto]: + if not registry_proto: + return None + for pm in registry_proto.project_metadata: + if pm.project == project: + return pm + return None + + +def _init_project_metadata(cached_registry_proto: RegistryProto, project: str): + new_project_uuid = f"{uuid.uuid4()}" + usage.set_current_project_uuid(new_project_uuid) + cached_registry_proto.project_metadata.append( + ProjectMetadata(project_name=project, project_uuid=new_project_uuid).to_proto() + ) + + class Registry(BaseRegistry): """ Registry: A registry allows for the management and persistence of feature definitions and related metadata. @@ -811,13 +850,14 @@ def clone(self) -> "Registry": new_registry._registry_store = NoopRegistryStore() return new_registry - def _initialize_registry(self): + def _initialize_registry(self, project: str): """Explicitly initializes the registry with an empty proto if it doesn't exist.""" try: - self._get_registry_proto() + self._get_registry_proto(project=project) except FileNotFoundError: registry_proto = RegistryProto() registry_proto.registry_schema_version = REGISTRY_SCHEMA_VERSION + _init_project_metadata(registry_proto, project) self._registry_store.update_registry_proto(registry_proto) def update_infra(self, infra: Infra, project: str, commit: bool = True): @@ -829,7 +869,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True): project: Feast project that the Infra object refers to commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto self.cached_registry_proto.infra.CopyFrom(infra.to_proto()) @@ -847,7 +887,9 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra: Returns: The stored Infra object. """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) return Infra.from_proto(registry_proto.infra) def apply_entity(self, entity: Entity, project: str, commit: bool = True): @@ -868,7 +910,7 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): entity_proto = entity.to_proto() entity_proto.spec.project = project - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, existing_entity_proto in enumerate( @@ -896,7 +938,9 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] Returns: List of entities """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) entities = [] for entity_proto in registry_proto.entities: if entity_proto.spec.project == project: @@ -916,7 +960,9 @@ def list_data_sources( Returns: List of data sources """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) data_sources = [] for data_source_proto in registry_proto.data_sources: if data_source_proto.project == project: @@ -934,7 +980,7 @@ def apply_data_source( project: Feast project that this data source belongs to commit: Whether to immediately commit to the registry """ - registry = self._prepare_registry_for_changes() + registry = self._prepare_registry_for_changes(project) for idx, existing_data_source_proto in enumerate(registry.data_sources): if existing_data_source_proto.name == data_source.name: del registry.data_sources[idx] @@ -959,7 +1005,7 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): project: Feast project that this data source belongs to commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, data_source_proto in enumerate( @@ -990,7 +1036,7 @@ def apply_feature_service( feature_service_proto = feature_service.to_proto() feature_service_proto.spec.project = project - registry = self._prepare_registry_for_changes() + registry = self._prepare_registry_for_changes(project) for idx, existing_feature_service_proto in enumerate(registry.feature_services): if ( @@ -1017,7 +1063,7 @@ def list_feature_services( List of feature services """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) feature_services = [] for feature_service_proto in registry.feature_services: if feature_service_proto.spec.project == project: @@ -1041,7 +1087,7 @@ def get_feature_service( Returns either the specified feature service, or raises an exception if none is found """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) for feature_service_proto in registry.feature_services: if ( @@ -1064,7 +1110,9 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti Returns either the specified entity, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) for entity_proto in registry_proto.entities: if entity_proto.spec.name == name and entity_proto.spec.project == project: return Entity.from_proto(entity_proto) @@ -1090,7 +1138,7 @@ def apply_feature_view( feature_view_proto = feature_view.to_proto() feature_view_proto.spec.project = project - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto self._check_conflicting_feature_view_names(feature_view) @@ -1146,7 +1194,7 @@ def list_stream_feature_views( Returns: List of stream feature views """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) stream_feature_views = [] for stream_feature_view in registry.stream_feature_views: if stream_feature_view.spec.project == project: @@ -1169,7 +1217,7 @@ def list_on_demand_feature_views( List of on demand feature views """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) on_demand_feature_views = [] for on_demand_feature_view in registry.on_demand_feature_views: if on_demand_feature_view.spec.project == project: @@ -1193,7 +1241,7 @@ def get_on_demand_feature_view( Returns either the specified on demand feature view, or raises an exception if none is found """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) for on_demand_feature_view in registry.on_demand_feature_views: if ( @@ -1217,7 +1265,7 @@ def get_data_source( Returns: Returns either the specified data source, or raises an exception if none is found """ - registry = self._get_registry_proto(allow_cache=allow_cache) + registry = self._get_registry_proto(project=project, allow_cache=allow_cache) for data_source in registry.data_sources: if data_source.project == project and data_source.name == name: @@ -1242,7 +1290,7 @@ def apply_materialization( end_date (datetime): End date of the materialization interval to track commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, existing_feature_view_proto in enumerate( @@ -1306,7 +1354,9 @@ def list_feature_views( Returns: List of feature views """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) feature_views: List[FeatureView] = [] for feature_view_proto in registry_proto.feature_views: if feature_view_proto.spec.project == project: @@ -1320,13 +1370,12 @@ def get_request_feature_view(self, name: str, project: str): Args: name: Name of feature view project: Feast project that this feature view belongs to - allow_cache: Allow returning feature view from the cached registry Returns: Returns either the specified feature view, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=False) + registry_proto = self._get_registry_proto(project=project, allow_cache=False) for feature_view_proto in registry_proto.feature_views: if ( feature_view_proto.spec.name == name @@ -1348,7 +1397,9 @@ def list_request_feature_views( Returns: List of feature views """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) feature_views: List[RequestFeatureView] = [] for request_feature_view_proto in registry_proto.request_feature_views: if request_feature_view_proto.spec.project == project: @@ -1372,7 +1423,9 @@ def get_feature_view( Returns either the specified feature view, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) for feature_view_proto in registry_proto.feature_views: if ( feature_view_proto.spec.name == name @@ -1396,7 +1449,9 @@ def get_stream_feature_view( Returns either the specified feature view, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) for feature_view_proto in registry_proto.stream_feature_views: if ( feature_view_proto.spec.name == name @@ -1414,7 +1469,7 @@ def delete_feature_service(self, name: str, project: str, commit: bool = True): project: Feast project that this feature service belongs to commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, feature_service_proto in enumerate( @@ -1439,7 +1494,7 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, existing_feature_view_proto in enumerate( @@ -1501,7 +1556,7 @@ def delete_entity(self, name: str, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, existing_entity_proto in enumerate( @@ -1536,7 +1591,7 @@ def apply_saved_dataset( saved_dataset_proto = saved_dataset.to_proto() saved_dataset_proto.spec.project = project - self._prepare_registry_for_changes() + self._prepare_registry_for_changes(project) assert self.cached_registry_proto for idx, existing_saved_dataset_proto in enumerate( @@ -1568,7 +1623,9 @@ def get_saved_dataset( Returns either the specified SavedDataset, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) for saved_dataset in registry_proto.saved_datasets: if ( saved_dataset.spec.name == name @@ -1590,7 +1647,9 @@ def list_saved_datasets( Returns: Returns the list of SavedDatasets """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) return [ SavedDataset.from_proto(saved_dataset) for saved_dataset in registry_proto.saved_datasets @@ -1614,7 +1673,7 @@ def apply_validation_reference( validation_reference_proto = validation_reference.to_proto() validation_reference_proto.project = project - registry_proto = self._prepare_registry_for_changes() + registry_proto = self._prepare_registry_for_changes(project) for idx, existing_validation_reference in enumerate( registry_proto.validation_references ): @@ -1644,7 +1703,9 @@ def get_validation_reference( Returns either the specified ValidationReference, or raises an exception if none is found """ - registry_proto = self._get_registry_proto(allow_cache=allow_cache) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) for validation_reference in registry_proto.validation_references: if ( validation_reference.name == name @@ -1662,7 +1723,7 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr project: Feast project that this object belongs to commit: Whether the change should be persisted immediately """ - registry_proto = self._prepare_registry_for_changes() + registry_proto = self._prepare_registry_for_changes(project) for idx, existing_validation_reference in enumerate( registry_proto.validation_references ): @@ -1676,14 +1737,26 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr return raise ValidationReferenceNotFound(name, project=project) + def list_project_metadata( + self, project: str, allow_cache: bool = False + ) -> List[ProjectMetadata]: + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return [ + ProjectMetadata.from_proto(project_metadata) + for project_metadata in registry_proto.project_metadata + if project_metadata.project == project + ] + def commit(self): """Commits the state of the registry cache to the remote registry store.""" if self.cached_registry_proto: self._registry_store.update_registry_proto(self.cached_registry_proto) - def refresh(self): + def refresh(self, project: Optional[str]): """Refreshes the state of the registry cache by fetching the registry state from the remote registry store.""" - self._get_registry_proto(allow_cache=False) + self._get_registry_proto(project=project, allow_cache=False) def teardown(self): """Tears down (removes) the registry.""" @@ -1692,21 +1765,34 @@ def teardown(self): def proto(self) -> RegistryProto: return self.cached_registry_proto or RegistryProto() - def _prepare_registry_for_changes(self): + def _prepare_registry_for_changes(self, project: str): """Prepares the Registry for changes by refreshing the cache if necessary.""" try: - self._get_registry_proto(allow_cache=True) + self._get_registry_proto(project=project, allow_cache=True) + if _get_project_metadata(self.cached_registry_proto, project) is None: + # Project metadata not initialized yet. Try pulling without cache + self._get_registry_proto(project=project, allow_cache=False) except FileNotFoundError: registry_proto = RegistryProto() registry_proto.registry_schema_version = REGISTRY_SCHEMA_VERSION self.cached_registry_proto = registry_proto self.cached_registry_proto_created = datetime.utcnow() + + # Initialize project metadata if needed + assert self.cached_registry_proto + if _get_project_metadata(self.cached_registry_proto, project) is None: + _init_project_metadata(self.cached_registry_proto, project) + self.commit() + return self.cached_registry_proto - def _get_registry_proto(self, allow_cache: bool = False) -> RegistryProto: + def _get_registry_proto( + self, project: Optional[str], allow_cache: bool = False + ) -> RegistryProto: """Returns the cached or remote registry state Args: + project: Name of the Feast project (optional) allow_cache: Whether to allow the use of the registry cache when fetching the RegistryProto Returns: Returns a RegistryProto object which represents the state of the registry @@ -1727,7 +1813,15 @@ def _get_registry_proto(self, allow_cache: bool = False) -> RegistryProto: ) ) - if allow_cache and not expired: + if project: + old_project_metadata = _get_project_metadata( + registry_proto=self.cached_registry_proto, project=project + ) + + if allow_cache and not expired and old_project_metadata is not None: + assert isinstance(self.cached_registry_proto, RegistryProto) + return self.cached_registry_proto + elif allow_cache and not expired: assert isinstance(self.cached_registry_proto, RegistryProto) return self.cached_registry_proto @@ -1735,6 +1829,18 @@ def _get_registry_proto(self, allow_cache: bool = False) -> RegistryProto: self.cached_registry_proto = registry_proto self.cached_registry_proto_created = datetime.utcnow() + if not project: + return registry_proto + + project_metadata = _get_project_metadata( + registry_proto=registry_proto, project=project + ) + if project_metadata: + usage.set_current_project_uuid(project_metadata.project_uuid) + else: + _init_project_metadata(registry_proto, project) + self.commit() + return registry_proto def _check_conflicting_feature_view_names(self, feature_view: BaseFeatureView): diff --git a/sdk/python/feast/templates/gcp/driver_repo.py b/sdk/python/feast/templates/gcp/driver_repo.py index acb17d55197..6c904a0fee7 100644 --- a/sdk/python/feast/templates/gcp/driver_repo.py +++ b/sdk/python/feast/templates/gcp/driver_repo.py @@ -18,6 +18,7 @@ # Indicates a data source from which feature values can be retrieved. Sources are queried when building training # datasets or materializing features into an online store. driver_stats_source = BigQuerySource( + name="driver_hourly_stats_source", # The BigQuery table where features can be found table="feast-oss.demo_data.driver_hourly_stats_2", # The event timestamp is used for point-in-time joins and for ensuring only diff --git a/sdk/python/feast/templates/hbase/example.py b/sdk/python/feast/templates/hbase/example.py index b34696185b1..6845371f1f2 100644 --- a/sdk/python/feast/templates/hbase/example.py +++ b/sdk/python/feast/templates/hbase/example.py @@ -9,6 +9,7 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( + name="driver_hourly_stats_source", path="%PARQUET_PATH%", timestamp_field="event_timestamp", created_timestamp_column="created", diff --git a/sdk/python/feast/templates/local/example.py b/sdk/python/feast/templates/local/example.py index 30f9adf1890..4fd30ba3a17 100644 --- a/sdk/python/feast/templates/local/example.py +++ b/sdk/python/feast/templates/local/example.py @@ -9,6 +9,7 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( + name="driver_hourly_stats_source", path="%PARQUET_PATH%", timestamp_field="event_timestamp", created_timestamp_column="created", diff --git a/sdk/python/feast/usage.py b/sdk/python/feast/usage.py index 90b659479d1..471a1b96719 100644 --- a/sdk/python/feast/usage.py +++ b/sdk/python/feast/usage.py @@ -40,6 +40,7 @@ _is_enabled = os.getenv(FEAST_USAGE, default=DEFAULT_FEAST_USAGE_VALUE) == "True" _constant_attributes = { + "project_id": "", "session_id": str(uuid.uuid4()), "installation_id": None, "version": get_version(), @@ -53,6 +54,10 @@ } +def set_current_project_uuid(project_uuid: str): + _constant_attributes["project_id"] = project_uuid + + @dataclasses.dataclass class FnCall: fn_name: str diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 4dc1db4a135..26423a9db16 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -421,7 +421,7 @@ def construct_test_environment( fs = FeatureStore(repo_dir_name) # We need to initialize the registry, because if nothing is applied in the test before tearing down # the feature store, that will cause the teardown method to blow up. - fs.registry._initialize_registry() + fs.registry._initialize_registry(project) environment = Environment( name=project, test_repo_config=test_repo_config, diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index ac7696f6e7f..27bbbbd2bb8 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -110,7 +110,7 @@ def test_apply_entity_success(test_registry): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.integration @@ -150,7 +150,7 @@ def test_apply_entity_integration(test_registry): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.parametrize( @@ -225,7 +225,7 @@ def test_apply_feature_view_success(test_registry): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.parametrize( @@ -299,7 +299,7 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.parametrize( @@ -364,7 +364,7 @@ def simple_udf(x: int): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.parametrize( @@ -488,7 +488,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.integration @@ -564,7 +564,7 @@ def test_apply_feature_view_integration(test_registry): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) @pytest.mark.integration @@ -640,7 +640,7 @@ def run_test_data_source_apply(test_registry: Registry): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) def test_commit(): @@ -656,10 +656,15 @@ def test_commit(): # Register Entity without commiting test_registry.apply_entity(entity, project, commit=False) + assert test_registry.cached_registry_proto + assert len(test_registry.cached_registry_proto.project_metadata) == 1 + project_metadata = test_registry.cached_registry_proto.project_metadata[0] + project_uuid = project_metadata.project_uuid + assert len(project_uuid) == 36 + assert_project_uuid(project_uuid, test_registry) # Retrieving the entity should still succeed entities = test_registry.list_entities(project, allow_cache=True) - entity = entities[0] assert ( len(entities) == 1 @@ -668,6 +673,7 @@ def test_commit(): and "team" in entity.tags and entity.tags["team"] == "matchmaking" ) + assert_project_uuid(project_uuid, test_registry) entity = test_registry.get_entity("driver_car_id", project, allow_cache=True) assert ( @@ -676,6 +682,7 @@ def test_commit(): and "team" in entity.tags and entity.tags["team"] == "matchmaking" ) + assert_project_uuid(project_uuid, test_registry) # Create new registry that points to the same store registry_with_same_store = Registry(registry_config, None) @@ -683,6 +690,7 @@ def test_commit(): # Retrieving the entity should fail since the store is empty entities = registry_with_same_store.list_entities(project) assert len(entities) == 0 + assert_project_uuid(project_uuid, registry_with_same_store) # commit from the original registry test_registry.commit() @@ -692,7 +700,6 @@ def test_commit(): # Retrieving the entity should now succeed entities = registry_with_same_store.list_entities(project) - entity = entities[0] assert ( len(entities) == 1 @@ -701,6 +708,7 @@ def test_commit(): and "team" in entity.tags and entity.tags["team"] == "matchmaking" ) + assert_project_uuid(project_uuid, registry_with_same_store) entity = test_registry.get_entity("driver_car_id", project) assert ( @@ -714,4 +722,10 @@ def test_commit(): # Will try to reload registry, which will fail because the file has been deleted with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto() + test_registry._get_registry_proto(project=project) + + +def assert_project_uuid(project_uuid, test_registry): + assert len(test_registry.cached_registry_proto.project_metadata) == 1 + project_metadata = test_registry.cached_registry_proto.project_metadata[0] + assert project_metadata.project_uuid == project_uuid diff --git a/sdk/python/tests/integration/registration/test_sql_registry.py b/sdk/python/tests/integration/registration/test_sql_registry.py index c483a7c46f6..56aff8c6d19 100644 --- a/sdk/python/tests/integration/registration/test_sql_registry.py +++ b/sdk/python/tests/integration/registration/test_sql_registry.py @@ -117,8 +117,14 @@ def test_apply_entity_success(sql_registry): # Register Entity sql_registry.apply_entity(entity, project) + project_metadata = sql_registry.list_project_metadata(project=project) + assert len(project_metadata) == 1 + project_uuid = project_metadata[0].project_uuid + assert len(project_metadata[0].project_uuid) == 36 + assert_project_uuid(project, project_uuid, sql_registry) entities = sql_registry.list_entities(project) + assert_project_uuid(project, project_uuid, sql_registry) entity = entities[0] assert ( @@ -138,12 +144,20 @@ def test_apply_entity_success(sql_registry): ) sql_registry.delete_entity("driver_car_id", project) + assert_project_uuid(project, project_uuid, sql_registry) entities = sql_registry.list_entities(project) + assert_project_uuid(project, project_uuid, sql_registry) assert len(entities) == 0 sql_registry.teardown() +def assert_project_uuid(project, project_uuid, sql_registry): + project_metadata = sql_registry.list_project_metadata(project=project) + assert len(project_metadata) == 1 + assert project_metadata[0].project_uuid == project_uuid + + @pytest.mark.skipif( sys.platform == "darwin" and "GITHUB_REF" in os.environ, reason="does not run on mac github actions", From 6645efb012f25580214724413875a0e77ab37d7f Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 11 Jul 2022 17:35:43 -0700 Subject: [PATCH 23/26] ci: Add a nightly CI job for integration tests (#2652) * Remove 3.7 wheels Signed-off-by: Kevin Zhang * ci: Add a nightly CI job for integration tests Signed-off-by: Achal Shah * Update and rebase Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang Co-authored-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 4 +- .github/workflows/nightly-ci.yml | 140 +++++++++++++++++++++ .github/workflows/pr_integration_tests.yml | 2 +- 3 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/nightly-ci.yml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 30b40df3b35..4a6bc34d093 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -66,7 +66,7 @@ jobs: uses: pypa/cibuildwheel@v2.7.0 env: CIBW_BUILD: "cp3*_x86_64" - CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" + CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64" CIBW_ARCHS: "native" CIBW_ENVIRONMENT: > COMPILE_GO=True PATH=$PATH:/usr/local/go/bin @@ -150,7 +150,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-10.15 ] - python-version: [ "3.7", "3.8", "3.9", "3.10"] + python-version: [ "3.8", "3.9", "3.10"] from-source: [ True, False ] env: # this script is for testing servers diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml new file mode 100644 index 00000000000..06571c2dd3f --- /dev/null +++ b/.github/workflows/nightly-ci.yml @@ -0,0 +1,140 @@ +name: nightly-ci + +on: + schedule: + - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. + +# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 +#concurrency: +# group: pr-integration-tests-${{ github.event.pull_request.number }} +# cancel-in-progress: true + +jobs: + check_date: + runs-on: ubuntu-latest + name: Check latest commit + outputs: + WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }} + steps: + - uses: actions/checkout@v2 + with: + ref: develop + - id: check_date + name: Check if there were commits in the last day + if: ${{ github.event_name == 'schedule' }} + run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') + + integration-test-python: + needs: [check_date] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.8" ] + os: [ ubuntu-latest ] + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + with: + # pull_request_target runs the workflow in the context of the base repo + # as such actions/checkout needs to be explicit configured to retrieve + # code from the PR. + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v2 + id: setup-python + with: + python-version: ${{ os.PYTHON }} + architecture: x64 + - name: Setup Go + id: setup-go + uses: actions/setup-go@v2 + with: + go-version: 1.18.0 + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Use AWS CLI + run: aws sts get-caller-identity + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<22.1" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.pip-cache.outputs.dir }} + /opt/hostedtoolcache/Python + /Users/runner/hostedtoolcache/Python + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + restore-keys: | + ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Install pip-tools + run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Setup Redis Cluster + run: | + docker pull vishnunair/docker-redis-cluster:latest + docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster + - name: Test python + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} + FEAST_USAGE: "False" + IS_TEST: "True" + SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} + SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} + SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} + SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} + SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} + run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: integrationtests + env_vars: OS,PYTHON + fail_ci_if_error: true + verbose: true \ No newline at end of file diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index db9e48fc2d8..8f64950a300 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -191,4 +191,4 @@ jobs: flags: integrationtests env_vars: OS,PYTHON fail_ci_if_error: true - verbose: true + verbose: true \ No newline at end of file From b35553b7db27c1f332dca836ce96d8c30d1100fb Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Jul 2022 10:02:38 -0700 Subject: [PATCH 24/26] fix: Fix night ci syntax error and update readme (#2935) * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 4 ++-- README.md | 2 +- docs/roadmap.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 06571c2dd3f..f125ebdc233 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -25,7 +25,7 @@ jobs: run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') integration-test-python: - needs: [check_date] + needs: check_date runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -45,7 +45,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - steps: + steps: - uses: actions/checkout@v2 with: # pull_request_target runs the workflow in the context of the base repo diff --git a/README.md b/README.md index 6bd7c95b7dc..64f6c5940fb 100644 --- a/README.md +++ b/README.md @@ -230,4 +230,4 @@ Thanks goes to these incredible people: - + \ No newline at end of file diff --git a/docs/roadmap.md b/docs/roadmap.md index 19af4f95c91..8461256a15b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -59,7 +59,7 @@ The list below contains the functionality that contributors are planning to deve * [ ] Java Client * [ ] Go Client * [ ] Delete API - * [] Feature Logging (for training) + * [ ] Feature Logging (for training) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * [ ] Training-serving skew detection (in progress) From c36361951d29714392b1def6e54f83ae45cd5d9a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 11:36:22 -0700 Subject: [PATCH 25/26] fix: Fix nightly ci again (#2939) * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 67 ++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index f125ebdc233..fead5124081 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -18,14 +18,68 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: develop + ref: master - id: check_date name: Check if there were commits in the last day if: ${{ github.event_name == 'schedule' }} run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') - + build-docker-image: + needs: [check_date] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: master + submodules: recursive + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" + - name: Cache Public ECR Image + id: lambda_python_3_9 + uses: actions/cache@v2 + with: + path: ~/cache + key: lambda_python_3_9 + - name: Handle Cache Miss (pull public ECR image & save it to tar file) + if: steps.cache-primes.outputs.cache-hit != 'true' + run: | + mkdir -p ~/cache + docker pull public.ecr.aws/lambda/python:3.9 + docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + - name: Handle Cache Hit (load docker image from tar file) + if: steps.cache-primes.outputs.cache-hit == 'true' + run: | + docker load -i ~/cache/lambda_python_3_9.tar + - name: Build and push + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: feast-python-server + run: | + docker build \ + --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + --load \ + . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: - needs: check_date + needs: [check_date, build-docker-image] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -48,16 +102,13 @@ jobs: steps: - uses: actions/checkout@v2 with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge + ref: master submodules: recursive - name: Setup Python uses: actions/setup-python@v2 id: setup-python with: - python-version: ${{ os.PYTHON }} + python-version: ${{ matrix.python-version }} architecture: x64 - name: Setup Go id: setup-go From 0bfc7d9d510cbcae70933ae420c832fcafd4dbe1 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Tue, 19 Jul 2022 00:59:55 +0000 Subject: [PATCH 26/26] chore(release): release 0.22.1 ## [0.22.1](https://github.com/feast-dev/feast/compare/v0.22.0...v0.22.1) (2022-07-19) ### Bug Fixes * Change numpy version on setup.py and upgrade it to resolve dependabot warning ([#2887](https://github.com/feast-dev/feast/issues/2887)) ([b9190b9](https://github.com/feast-dev/feast/commit/b9190b9f37a9a05421c3a288e9ba1807124dcb12)) * Change the feature store plan method to public modifier ([#2904](https://github.com/feast-dev/feast/issues/2904)) ([568058a](https://github.com/feast-dev/feast/commit/568058aee02e4e8b961caf312e3f1c7ff909a3f2)) * Deprecate 3.7 wheels and fix verification workflow ([#2934](https://github.com/feast-dev/feast/issues/2934)) ([146e36d](https://github.com/feast-dev/feast/commit/146e36df5809f76d00900d0ee787bf9aa3f078bd)) * Fix build wheels workflow to install apache-arrow correctly ([#2932](https://github.com/feast-dev/feast/issues/2932)) ([4b69e0e](https://github.com/feast-dev/feast/commit/4b69e0e839e4fdc6268ef077f02bd5b727c44370)) * Fix grpc and update protobuf ([#2894](https://github.com/feast-dev/feast/issues/2894)) ([f726c96](https://github.com/feast-dev/feast/commit/f726c967dc132902423d336d990715c3033fe196)) * Fix night ci syntax error and update readme ([#2935](https://github.com/feast-dev/feast/issues/2935)) ([b35553b](https://github.com/feast-dev/feast/commit/b35553b7db27c1f332dca836ce96d8c30d1100fb)) * Fix nightly ci again ([#2939](https://github.com/feast-dev/feast/issues/2939)) ([c363619](https://github.com/feast-dev/feast/commit/c36361951d29714392b1def6e54f83ae45cd5d9a)) * Fix the go build and use CgoArrowAllocator to prevent incorrect garbage collection ([#2919](https://github.com/feast-dev/feast/issues/2919)) ([f4f4894](https://github.com/feast-dev/feast/commit/f4f4894d1db4b2192a502999f87711b0529e6e83)) * Fixing broken links to feast documentation on java readme and contribution ([#2892](https://github.com/feast-dev/feast/issues/2892)) ([a45e10a](https://github.com/feast-dev/feast/commit/a45e10a2558129fcb906fe3e7a03006aeea49d97)) * Resolve small typo in README file ([#2930](https://github.com/feast-dev/feast/issues/2930)) ([9840c1b](https://github.com/feast-dev/feast/commit/9840c1b4c64898fcb06e34dcea14becce8af44c3)) * Update gopy to point to fork to resolve github annotation errors. ([#2940](https://github.com/feast-dev/feast/issues/2940)) ([9b9fbbe](https://github.com/feast-dev/feast/commit/9b9fbbe692475d85b713b7e6707a105fca35fbf9)) --- CHANGELOG.md | 17 +++++++++++++++++ infra/charts/feast-python-server/Chart.yaml | 2 +- infra/charts/feast-python-server/README.md | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- .../feast/charts/feature-server/Chart.yaml | 4 ++-- .../feast/charts/feature-server/README.md | 4 ++-- .../feast/charts/feature-server/values.yaml | 2 +- .../charts/transformation-service/Chart.yaml | 4 ++-- .../charts/transformation-service/README.md | 4 ++-- .../charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- 13 files changed, 36 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd7e8098f36..f49d066770e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [0.22.1](https://github.com/feast-dev/feast/compare/v0.22.0...v0.22.1) (2022-07-19) + + +### Bug Fixes + +* Change numpy version on setup.py and upgrade it to resolve dependabot warning ([#2887](https://github.com/feast-dev/feast/issues/2887)) ([b9190b9](https://github.com/feast-dev/feast/commit/b9190b9f37a9a05421c3a288e9ba1807124dcb12)) +* Change the feature store plan method to public modifier ([#2904](https://github.com/feast-dev/feast/issues/2904)) ([568058a](https://github.com/feast-dev/feast/commit/568058aee02e4e8b961caf312e3f1c7ff909a3f2)) +* Deprecate 3.7 wheels and fix verification workflow ([#2934](https://github.com/feast-dev/feast/issues/2934)) ([146e36d](https://github.com/feast-dev/feast/commit/146e36df5809f76d00900d0ee787bf9aa3f078bd)) +* Fix build wheels workflow to install apache-arrow correctly ([#2932](https://github.com/feast-dev/feast/issues/2932)) ([4b69e0e](https://github.com/feast-dev/feast/commit/4b69e0e839e4fdc6268ef077f02bd5b727c44370)) +* Fix grpc and update protobuf ([#2894](https://github.com/feast-dev/feast/issues/2894)) ([f726c96](https://github.com/feast-dev/feast/commit/f726c967dc132902423d336d990715c3033fe196)) +* Fix night ci syntax error and update readme ([#2935](https://github.com/feast-dev/feast/issues/2935)) ([b35553b](https://github.com/feast-dev/feast/commit/b35553b7db27c1f332dca836ce96d8c30d1100fb)) +* Fix nightly ci again ([#2939](https://github.com/feast-dev/feast/issues/2939)) ([c363619](https://github.com/feast-dev/feast/commit/c36361951d29714392b1def6e54f83ae45cd5d9a)) +* Fix the go build and use CgoArrowAllocator to prevent incorrect garbage collection ([#2919](https://github.com/feast-dev/feast/issues/2919)) ([f4f4894](https://github.com/feast-dev/feast/commit/f4f4894d1db4b2192a502999f87711b0529e6e83)) +* Fixing broken links to feast documentation on java readme and contribution ([#2892](https://github.com/feast-dev/feast/issues/2892)) ([a45e10a](https://github.com/feast-dev/feast/commit/a45e10a2558129fcb906fe3e7a03006aeea49d97)) +* Resolve small typo in README file ([#2930](https://github.com/feast-dev/feast/issues/2930)) ([9840c1b](https://github.com/feast-dev/feast/commit/9840c1b4c64898fcb06e34dcea14becce8af44c3)) +* Update gopy to point to fork to resolve github annotation errors. ([#2940](https://github.com/feast-dev/feast/issues/2940)) ([9b9fbbe](https://github.com/feast-dev/feast/commit/9b9fbbe692475d85b713b7e6707a105fca35fbf9)) + # [0.22.0](https://github.com/feast-dev/feast/compare/v0.21.0...v0.22.0) (2022-06-29) diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index 6c4751e3b7e..661701e164d 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.22.0 +version: 0.22.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index 3f60cc6c549..0c1a1000a6b 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -1,6 +1,6 @@ # feast-python-server -![Version: 0.22.0](https://img.shields.io/badge/Version-0.22.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.22.1](https://img.shields.io/badge/Version-0.22.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) Feast Feature Server in Python diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 012dc47de92..c2d35bb2169 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.22.0 +version: 0.22.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 8b5e9718eff..e9e66a7e9aa 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast components that are being installed on ## Chart: Feast -Feature store for machine learning Current chart version is `0.22.0` +Feature store for machine learning Current chart version is `0.22.1` ## Installation @@ -55,8 +55,8 @@ For more details, please see: https://docs.feast.dev/how-to-guides/running-feast | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.22.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.22.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.22.1 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.22.1 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index b88ad599b4b..b61675c0f5f 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.22.0 -appVersion: v0.22.0 +version: 0.22.1 +appVersion: v0.22.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 28570b0fc67..7aac6d1296e 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.22.0](https://img.shields.io/badge/Version-0.22.0-informational?style=flat-square) ![AppVersion: v0.22.0](https://img.shields.io/badge/AppVersion-v0.22.0-informational?style=flat-square) +![Version: 0.22.1](https://img.shields.io/badge/Version-0.22.1-informational?style=flat-square) ![AppVersion: v0.22.1](https://img.shields.io/badge/AppVersion-v0.22.1-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.22.0"` | Image tag | +| image.tag | string | `"0.22.1"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index df8367fede2..ee1c65efac4 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.22.0 + tag: 0.22.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 148e136acf3..d48e2b9358b 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.22.0 -appVersion: v0.22.0 +version: 0.22.1 +appVersion: v0.22.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4cbc1048f64..d940e6181dc 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.22.0](https://img.shields.io/badge/Version-0.22.0-informational?style=flat-square) ![AppVersion: v0.22.0](https://img.shields.io/badge/AppVersion-v0.22.0-informational?style=flat-square) +![Version: 0.22.1](https://img.shields.io/badge/Version-0.22.1-informational?style=flat-square) ![AppVersion: v0.22.1](https://img.shields.io/badge/AppVersion-v0.22.1-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.22.0"` | Image tag | +| image.tag | string | `"0.22.1"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 1264ea4f7b8..55e8f75f4fa 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.22.0 + tag: 0.22.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 0b69f295e7b..bf8a1d0b553 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.22.0 + version: 0.22.1 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.22.0 + version: 0.22.1 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 7ea4bc07bd1..a1e10a040bd 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -38,7 +38,7 @@ - 0.22.0 + 0.22.1 https://github.com/feast-dev/feast UTF-8