From a21bb621c915c9984f43e2c6867c8b5e0b516027 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 23 Apr 2024 16:33:16 +0400 Subject: [PATCH 01/11] chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni --- .devcontainer/devcontainer.json | 5 +---- .../fork_pr_integration_tests_aws.yml | 2 +- .../fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 6 +++--- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/unit_tests.yml | 4 ++-- .../java/feast/serving/it/ServingEnvironment.java | 7 ++----- .../docker-compose/docker-compose-redis-it.yml | 11 ++++------- .../resources/docker-compose/feast10/Dockerfile | 13 +++++++------ .../resources/docker-compose/feast10/entrypoint.sh | 4 ---- .../docker-compose/feast10/requirements.txt | 6 ------ 12 files changed, 23 insertions(+), 41 deletions(-) delete mode 100644 java/serving/src/test/resources/docker-compose/feast10/requirements.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b4da25737f9..e82fd04db4a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,10 +7,7 @@ }, "ghcr.io/devcontainers/features/python:1": { "version": "3.9" - }, - "ghcr.io/meaningful-ooo/devcontainer-features/homebrew:2": { - "version": "latest" } }, - "postCreateCommand": "brew install mysql && pip install -e '.[dev]' && make compile-protos-python" + "postCreateCommand": "pip install -e '.[dev]' && make compile-protos-python" } diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index be75c4f9875..52d7112a640 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -138,7 +138,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 0793fbd6e51..337d8040ae7 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -82,7 +82,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macOS-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index b9b6f8df06f..a3484a34625 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -72,7 +72,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f0851f5bb04..ca0a7dcfe25 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -79,7 +79,7 @@ jobs: build-source-distribution: name: Build source distribution - runs-on: macos-latest + runs-on: macos-12 steps: - uses: actions/checkout@v3 - name: Setup Python @@ -136,7 +136,7 @@ jobs: needs: [build-python-wheel, build-source-distribution, get-version] strategy: matrix: - os: [ubuntu-latest, macos-latest ] + os: [ubuntu-latest, macos-12 ] python-version: ["3.9", "3.10"] from-source: [ True, False ] env: @@ -165,7 +165,7 @@ jobs: name: wheels path: dist - name: Install OS X dependencies - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-12' run: brew install coreutils - name: Install wheel if: ${{ !matrix.from-source }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 4dea41d4ad0..3292d6bcb59 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -202,7 +202,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f3f91bb67f3..ff7c3d5e234 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,9 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest ] + os: [ ubuntu-latest, macos-12 ] exclude: - - os: macOS-latest + - os: macos-12 python-version: "3.9" env: OS: ${{ matrix.os }} diff --git a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java index 54892038566..356524399a4 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java +++ b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java @@ -62,11 +62,8 @@ static void globalSetup() { .withExposedService("redis", 6379) .withExposedService( "feast", 8080, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(180))) - .withTailChildContainers(true); - - if (System.getenv("FEAST_TESTCONTAINERS_LOCAL_COMPOSE") != null) { - environment = environment.withLocalCompose(true); - } + .withTailChildContainers(true) + .withLocalCompose(true); environment.start(); } diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml index 0522750d996..142efe7fa20 100644 --- a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml +++ b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml @@ -1,5 +1,3 @@ -version: '3' - services: redis: image: redis:6.2 @@ -7,11 +5,10 @@ services: ports: - "6379" feast: - build: feast10 + build: + context: ../../../../../../ + dockerfile: java/serving/src/test/resources/docker-compose/feast10/Dockerfile ports: - "8080" - links: + depends_on: - redis - volumes: - - $PWD/../../../../../../:/mnt/feast - diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 7e36658caef..8b3c5b3d3d4 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,12 +1,13 @@ FROM python:3.9 -WORKDIR /usr/src/ - -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt - WORKDIR /app -COPY . . +COPY java/serving/src/test/resources/docker-compose/feast10/ . +COPY sdk/python /mnt/feast/sdk/python +COPY protos /mnt/feast/protos +COPY setup.py /mnt/feast/setup.py +COPY pyproject.toml /mnt/feast/pyproject.toml +COPY README.md /mnt/feast/README.md +RUN cd /mnt/feast && SETUPTOOLS_SCM_PRETEND_VERSION="0.1.0" pip install .[grpcio,redis] EXPOSE 8080 CMD ["./entrypoint.sh"] diff --git a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh index 0690b734c38..82d9399521b 100755 --- a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh +++ b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh @@ -2,10 +2,6 @@ set -e -# feast root directory is expected to be mounted (eg, by docker compose) -cd /mnt/feast -pip install -e '.[grpcio,redis]' - cd /app python materialize.py feast serve_transformations --port 8080 diff --git a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt deleted file mode 100644 index 6ba2c53d817..00000000000 --- a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# for source generation -pyarrow==14.0.1 - -# temp fixes -proto-plus -Jinja2>=2.0.0 \ No newline at end of file From 94da29d0e38c787ea495436be9e0bb083439b13e Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Tue, 23 Apr 2024 08:20:25 -0500 Subject: [PATCH 02/11] chore: Incorporate release 0.37.1 to master (#4127) * chore(release): release 0.37.1 ## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) ### Bug Fixes * Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) ### Reverts * Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko --------- Signed-off-by: tokoko Co-authored-by: feast-ci-bot Co-authored-by: Tornike Gurgenidze Signed-off-by: Lokesh Rangineni --- CHANGELOG.md | 12 ++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 ++-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- infra/charts/feast/charts/feature-server/values.yaml | 2 +- .../feast/charts/transformation-service/Chart.yaml | 4 ++-- .../feast/charts/transformation-service/README.md | 4 ++-- .../feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 17 files changed, 39 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5fc0f01b1..19dc5d86d7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) + + +### Bug Fixes + +* Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) + + +### Reverts + +* Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) + # [0.37.0](https://github.com/feast-dev/feast/compare/v0.36.0...v0.37.0) (2024-04-17) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index bd4bc606a70..8d564f3b420 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.37.0 +version: 0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index fa6d89361c0..0730e39e63c 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.37.0` +Current chart version is `0.37.1` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.37.1"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 0de071ef3db..df5241ebb2d 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.37.0 + tag: 0.37.1 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 26a00d80c63..21c00e4483b 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.37.0 +version: 0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 47959047dc2..8ab816dc707 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.37.0` +Feature store for machine learning Current chart version is `0.37.1` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.1 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.1 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index f2f8c748dc4..08563c6e069 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.37.0 -appVersion: v0.37.0 +version: 0.37.1 +appVersion: v0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 531adce92da..3018b31c96c 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.37.1"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 1bf1a03f4a7..1d86059c1fd 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.37.0 + tag: 0.37.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 056e00473fb..bad9befa0bf 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.37.0 -appVersion: v0.37.0 +version: 0.37.1 +appVersion: v0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4b11861d539..f912b4c02f7 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.37.1"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index a04dfeb3e04..df5ea64c347 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.37.0 + tag: 0.37.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 9a2d3e0e807..80b8c861326 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.37.0 + version: 0.37.1 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.37.0 + version: 0.37.1 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 2d7e2c3e7d2..8ba8ed4ac53 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.37.0 + 0.37.1 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index cdad51e0608..61c89f648f2 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.37.0", + "@feast-dev/feast-ui": "0.37.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 07472d7bbeb..5d0101fbe52 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.37.0": - version "0.37.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.0.tgz#53e04a73835617f23528bd95228190cf022338b0" - integrity sha512-bpW4K6y+XE9np+/zXsYffQWrND7Bx3z1xBj4SSddgX3QkGJyaJeOM9Rfu4ZhkjwNGvfqKaxGz9Ucnhpjv9lrfw== +"@feast-dev/feast-ui@0.37.1": + version "0.37.1" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.1.tgz#b84618d1fd2e1dbc463ab2889964006b555d9ec4" + integrity sha512-xhHK3hWvW58ukB+kx04ut+7OIT+zuITw6eYKjuJmjzAZ2S8uVcqDso4T9Ma88qX+qhn4NWzNBUyM2Gz1xOhzKQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index 9209d7b03c7..ea69e571fb5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.37.0", + "version": "0.37.1", "private": false, "files": [ "dist" From 72f1e1bd872f99658c309242373af90115900510 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 24 Apr 2024 03:03:10 +0400 Subject: [PATCH 03/11] chore: Install python dependencies with uv in workflows (#4086) * install dependencies in unit-tests with uv Signed-off-by: tokoko * install dependencies in unit-tests with uv Signed-off-by: tokoko * enable caching, change linter job Signed-off-by: tokoko * change local integration tests to uv Signed-off-by: tokoko * change all installs to uv Signed-off-by: tokoko * try adding uv cache Signed-off-by: tokoko * fix lambda cache step name Signed-off-by: tokoko * reenable caches for uv Signed-off-by: tokoko * remove dangling cache step Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni --- .github/workflows/linter.yml | 22 ++--------- .github/workflows/master_only.yml | 31 +++++++--------- .github/workflows/nightly-ci.yml | 37 ++++++++----------- .github/workflows/pr_integration_tests.yml | 31 +++++++--------- .../workflows/pr_local_integration_tests.yml | 31 +++++++--------- .github/workflows/unit_tests.yml | 32 +++++++--------- Makefile | 5 +++ 7 files changed, 78 insertions(+), 111 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a1747db1356..f235b6f3bac 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -11,31 +11,17 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.9" architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv - name: Install dependencies run: | - make install-python-ci-dependencies + make install-python-ci-dependencies-uv - name: Lint python run: make lint-python diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 225f24a828a..c355c55c23e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Setup Python id: setup-python uses: actions/setup-python@v3 @@ -106,27 +106,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 3292d6bcb59..2db70879085 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -33,7 +33,7 @@ jobs: with: ref: master - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: "3.9" @@ -90,18 +90,18 @@ jobs: run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image id: lambda_python_3_9 - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/cache key: lambda_python_3_9 - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' + if: steps.lambda_python_3_9.outputs.cache-hit != 'true' run: | mkdir -p ~/cache docker pull public.ecr.aws/lambda/python:3.9 docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' + if: steps.lambda_python_3_9.outputs.cache-hit == 'true' run: | docker load -i ~/cache/lambda_python_3_9.tar - name: Build and push @@ -145,7 +145,7 @@ jobs: ref: master submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -173,25 +173,20 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5e7287351b6..4d28c6b456c 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -110,7 +110,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -133,27 +133,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 266cdcc9b9f..be892ae9106 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -33,32 +33,27 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak run: make test-python-integration-local diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ff7c3d5e234..5d689d72de0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,31 +19,27 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: | + pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test Python run: make test-python-unit diff --git a/Makefile b/Makefile index bf2d876b7f1..f42d5c1edaf 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,11 @@ install-python-ci-dependencies: pip install --no-deps -e . python setup.py build_python_protos --inplace +install-python-ci-dependencies-uv: + uv pip sync --system sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + uv pip install --system --no-deps -e . + python setup.py build_python_protos --inplace + lock-python-ci-dependencies: python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt From 3b6b72b0e80b70f4a119815f4dd62ee82611ad38 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 25 Apr 2024 09:46:12 +0400 Subject: [PATCH 04/11] feat: Make arrow primary interchange for online ODFV execution (#4143) * rewrite online flow to use transform_arrow Signed-off-by: tokoko * fix transformation server Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni --- sdk/python/feast/feature_store.py | 28 +++--- sdk/python/feast/on_demand_feature_view.py | 89 +------------------ sdk/python/feast/online_response.py | 11 +++ .../transformation/python_transformation.py | 3 - .../substrait_transformation.py | 3 - sdk/python/feast/transformation_server.py | 5 +- .../tests/unit/test_on_demand_feature_view.py | 2 +- 7 files changed, 29 insertions(+), 112 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index fafec32c5d5..e83a24b6644 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -2138,7 +2138,7 @@ def _augment_response_with_on_demand_transforms( ) initial_response = OnlineResponse(online_features_response) - initial_response_df: Optional[pd.DataFrame] = None + initial_response_arrow: Optional[pa.Table] = None initial_response_dict: Optional[Dict[str, List[Any]]] = None # Apply on demand transformations and augment the result rows @@ -2148,18 +2148,14 @@ def _augment_response_with_on_demand_transforms( if odfv.mode == "python": if initial_response_dict is None: initial_response_dict = initial_response.to_dict() - transformed_features_dict: Dict[str, List[Any]] = ( - odfv.get_transformed_features( - initial_response_dict, - full_feature_names, - ) + transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( + initial_response_dict ) elif odfv.mode in {"pandas", "substrait"}: - if initial_response_df is None: - initial_response_df = initial_response.to_df() - transformed_features_df: pd.DataFrame = odfv.get_transformed_features( - initial_response_df, - full_feature_names, + if initial_response_arrow is None: + initial_response_arrow = initial_response.to_arrow() + transformed_features_arrow = odfv.transform_arrow( + initial_response_arrow, full_feature_names ) else: raise Exception( @@ -2169,11 +2165,11 @@ def _augment_response_with_on_demand_transforms( transformed_features = ( transformed_features_dict if odfv.mode == "python" - else transformed_features_df + else transformed_features_arrow ) transformed_columns = ( - transformed_features.columns - if isinstance(transformed_features, pd.DataFrame) + transformed_features.column_names + if isinstance(transformed_features, pa.Table) else transformed_features ) selected_subset = [f for f in transformed_columns if f in _feature_refs] @@ -2183,6 +2179,10 @@ def _augment_response_with_on_demand_transforms( feature_vector = transformed_features[selected_feature] proto_values.append( python_values_to_proto_values(feature_vector, ValueType.UNKNOWN) + if odfv.mode == "python" + else python_values_to_proto_values( + feature_vector.to_numpy(), ValueType.UNKNOWN + ) ) odfv_result_names |= set(selected_subset) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index b532fa651a1..e5de1f20a65 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -12,7 +12,6 @@ from typeguard import typechecked from feast.base_feature_view import BaseFeatureView -from feast.batch_feature_view import BatchFeatureView from feast.data_source import RequestSource from feast.errors import RegistryInferenceFailure, SpecifiedFeaturesNotPresentError from feast.feature_view import FeatureView @@ -493,53 +492,7 @@ def transform_arrow( ] ) - def get_transformed_features_df( - self, - df_with_features: pd.DataFrame, - full_feature_names: bool = False, - ) -> pd.DataFrame: - # Apply on demand transformations - if not isinstance(df_with_features, pd.DataFrame): - raise TypeError("get_transformed_features_df only accepts pd.DataFrame") - columns_to_cleanup = [] - for source_fv_projection in self.source_feature_view_projections.values(): - for feature in source_fv_projection.features: - full_feature_ref = f"{source_fv_projection.name}__{feature.name}" - if full_feature_ref in df_with_features.keys(): - # Make sure the partial feature name is always present - df_with_features[feature.name] = df_with_features[full_feature_ref] - columns_to_cleanup.append(feature.name) - elif feature.name in df_with_features.keys(): - # Make sure the full feature name is always present - df_with_features[full_feature_ref] = df_with_features[feature.name] - columns_to_cleanup.append(full_feature_ref) - - # Compute transformed values and apply to each result row - df_with_transformed_features: pd.DataFrame = ( - self.feature_transformation.transform(df_with_features) - ) - - # Work out whether the correct columns names are used. - rename_columns: Dict[str, str] = {} - for feature in self.features: - short_name = feature.name - long_name = self._get_projected_feature_name(feature.name) - if ( - short_name in df_with_transformed_features.columns - and full_feature_names - ): - rename_columns[short_name] = long_name - elif not full_feature_names: - # Long name must be in dataframe. - rename_columns[long_name] = short_name - - # Cleanup extra columns used for transformation - df_with_transformed_features = df_with_transformed_features[ - [f.name for f in self.features] - ] - return df_with_transformed_features.rename(columns=rename_columns) - - def get_transformed_features_dict( + def transform_dict( self, feature_dict: Dict[str, Any], # type: ignore ) -> Dict[str, Any]: @@ -566,29 +519,6 @@ def get_transformed_features_dict( del output_dict[feature_name] return output_dict - def get_transformed_features( - self, - features: Union[Dict[str, Any], pd.DataFrame], - full_feature_names: bool = False, - ) -> Union[Dict[str, Any], pd.DataFrame]: - # TODO: classic inheritance pattern....maybe fix this - if self.mode == "python" and isinstance(features, Dict): - # note full_feature_names is not needed for the dictionary - return self.get_transformed_features_dict( - feature_dict=features, - ) - elif self.mode in {"pandas", "substrait"} and isinstance( - features, pd.DataFrame - ): - return self.get_transformed_features_df( - df_with_features=features, - full_feature_names=full_feature_names, - ) - else: - raise Exception( - f'Invalid OnDemandFeatureMode: {self.mode}. Expected one of "pandas" or "python".' - ) - def infer_features(self) -> None: inferred_features = self.feature_transformation.infer_features( self._construct_random_input() @@ -745,23 +675,6 @@ def decorator(user_function): return decorator -def feature_view_to_batch_feature_view(fv: FeatureView) -> BatchFeatureView: - bfv = BatchFeatureView( - name=fv.name, - entities=fv.entities, - ttl=fv.ttl, - tags=fv.tags, - online=fv.online, - owner=fv.owner, - schema=fv.schema, - source=fv.batch_source, - ) - - bfv.features = copy.copy(fv.features) - bfv.entities = copy.copy(fv.entities) - return bfv - - def _empty_odfv_udf_fn(x: Any) -> Any: # just an identity mapping, otherwise we risk tripping some downstream tests return x diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index 48524359bf3..050b374340e 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -15,6 +15,7 @@ from typing import Any, Dict, List import pandas as pd +import pyarrow as pa from feast.feature_view import DUMMY_ENTITY_ID from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesResponse @@ -77,3 +78,13 @@ def to_df(self, include_event_timestamps: bool = False) -> pd.DataFrame: """ return pd.DataFrame(self.to_dict(include_event_timestamps)) + + def to_arrow(self, include_event_timestamps: bool = False) -> pa.Table: + """ + Converts GetOnlineFeaturesResponse features into pyarrow Table. + + Args: + is_with_event_timestamps: bool Optionally include feature timestamps in the table + """ + + return pa.Table.from_pydict(self.to_dict(include_event_timestamps)) diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index ec950a24f3c..88cde7cc726 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -64,9 +64,6 @@ def __eq__(self, other): "Comparisons should only involve PythonTransformation class objects." ) - if not super().__eq__(other): - return False - if ( self.udf_string != other.udf_string or self.udf.__code__.co_code != other.udf.__code__.co_code diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index 48a87b62079..02b94d85726 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -77,9 +77,6 @@ def __eq__(self, other): "Comparisons should only involve SubstraitTransformation class objects." ) - if not super().__eq__(other): - return False - return ( self.substrait_plan == other.substrait_plan and self.ibis_function.__code__.co_code diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index 34fe3eac766..db8b0d942e2 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -45,15 +45,14 @@ def TransformFeatures(self, request, context): context.set_code(grpc.StatusCode.INVALID_ARGUMENT) raise - df = pa.ipc.open_file(request.transformation_input.arrow_value).read_pandas() + df = pa.ipc.open_file(request.transformation_input.arrow_value).read_all() if odfv.mode != "pandas": raise Exception( f'OnDemandFeatureView mode "{odfv.mode}" not supported by TransformationServer.' ) - result_df = odfv.get_transformed_features_df(df, True) - result_arrow = pa.Table.from_pandas(result_df) + result_arrow = odfv.transform_arrow(df, True) sink = pa.BufferOutputStream() writer = pa.ipc.new_file(sink, result_arrow.schema) writer.write_table(result_arrow) diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index cf4afa94228..402aa4e0e33 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -204,7 +204,7 @@ def test_python_native_transformation_mode(): } ) - assert on_demand_feature_view_python_native.get_transformed_features( + assert on_demand_feature_view_python_native.transform_dict( { "feature1": 0, "feature2": 1, From 7d0d05ae3309e0e2f8f3290c5c4ba3df454f926d Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Thu, 25 Apr 2024 05:15:19 -0700 Subject: [PATCH 05/11] feat: Feast/IKV online store documentation (#4146) * feat: Feast/IKV online store documentation Signed-off-by: Pushkar Gupta * functionality matric Signed-off-by: Pushkar Gupta * more changes Signed-off-by: Pushkar Gupta * mount dir Signed-off-by: Pushkar Gupta --------- Signed-off-by: Pushkar Gupta Signed-off-by: Lokesh Rangineni --- README.md | 1 + docs/SUMMARY.md | 1 + docs/reference/online-stores/README.md | 4 ++ docs/reference/online-stores/ikv.md | 73 ++++++++++++++++++++++++ docs/reference/online-stores/overview.md | 38 ++++++------ docs/roadmap.md | 1 + 6 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 docs/reference/online-stores/ikv.md diff --git a/README.md b/README.md index 6a851d0d417..aab9c332435 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) + * [x] [IKV - Inlined Key Value Store](https://docs.feast.dev/reference/online-stores/ikv) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8affea898ef..b211730d0ef 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -91,6 +91,7 @@ * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) * [Dragonfly](reference/online-stores/dragonfly.md) + * [IKV](reference/online-stores/ikv.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index d90bfcf1632..686e820f4e7 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -22,6 +22,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [dragonfly.md](dragonfly.md) {% endcontent-ref %} +{% content-ref url="ikv.md" %} +[ikv.md](ikv.md) +{% endcontent-ref %} + {% content-ref url="datastore.md" %} [datastore.md](datastore.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/ikv.md b/docs/reference/online-stores/ikv.md new file mode 100644 index 00000000000..ff690c1a622 --- /dev/null +++ b/docs/reference/online-stores/ikv.md @@ -0,0 +1,73 @@ +# IKV (Inlined Key-Value Store) online store + +## Description + +[IKV](https://github.com/inlinedio/ikv-store) is a fully-managed embedded key-value store, primarily designed for storing ML features. Most key-value stores (think Redis or Cassandra) need a remote database cluster, whereas IKV allows you to utilize your existing application infrastructure to store data (cost efficient) and access it without any network calls (better performance). + +For provisioning API keys for using it as an online-store in Feast, go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io + +## Getting started +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI: `pip install feast` + +In order to use this online store, you'll need to install the IKV extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, ikv]'` +- `pip install 'feast[snowflake, ikv]'` +- `pip install 'feast[aws, ikv]'` +- `pip install 'feast[azure, ikv]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in IKV as the online store as seen below in the examples. + +### 1. Provision an IKV store +Go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io + +### 2. Configure + +Update `my_feature_repo/feature_store.yaml` with the below contents: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: ikv + account_id: secret + account_passkey: secret + store_name: your-store-name + mount_directory: /absolute/path/on/disk/for/ikv/embedded/index +``` +{% endcode %} + +After provision an IKV account/store, you should the required id, passkey and store-name. + +Additionally you must specify a mount-directory - where IKV will pull/update (maintain) a copy of the index for online reads (IKV is an embedded database). It can be skipped only if you don't plan to read any data from this container. The mount directory path usually points to a location on local/remote disk. + +The full set of configuration options is available in IKVOnlineStoreConfig at `sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the IKV online store. + +| | IKV | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md index 7a51a9a4687..04d24447058 100644 --- a/docs/reference/online-stores/overview.md +++ b/docs/reference/online-stores/overview.md @@ -29,26 +29,26 @@ See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussio ## Functionality Matrix There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. -There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `IKVOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which online stores support what functionality. -| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | -| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | -| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | -| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Java | no | yes | no | no | no | no | no | no | -| readable by Go | yes | yes | no | no | no | no | no | no | -| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | -| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | -| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | -| support for deleting expired data | no | yes | no | no | no | no | no | no | -| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | -| collocated by feature service | no | no | no | no | no | no | no | no | -| collocated by entity key | no | yes | no | no | no | no | no | no | +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | [IKV](https://inlined.io) | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | +| collocated by feature service | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | diff --git a/docs/roadmap.md b/docs/roadmap.md index a04ede7c993..5ff262e3432 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,6 +33,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) + * [x] [IKV - Inlined Key Value Store](https://docs.feast.dev/reference/online-stores/ikv) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) From 0310a2ca0ba73ab68fcd6e6addbd13dec314708f Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:16:54 -0400 Subject: [PATCH 06/11] fix: Default value is not set in Redis connection string using environment variable (#4136) Removed documentation of Redis connection string supporting default values when using environment variables as it isn't supported Fixes #3669 Signed-off-by: Theodor Mihalache Signed-off-by: Lokesh Rangineni --- docs/how-to-guides/running-feast-in-production.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 9d1984d7366..4663c928c6e 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -257,17 +257,6 @@ online_store: connection_string: ${REDIS_CONNECTION_STRING} ``` -It is possible to set a default value if the environment variable is not set, with `${ENV_VAR:"default"}`. For instance: - -```yaml -project: my_project -registry: data/registry.db -provider: local -online_store: - type: redis - connection_string: ${REDIS_CONNECTION_STRING:"0.0.0.0:6379"} -``` - *** ## Summary From 786931edc77c24457cf582d1a3043e978aefd87f Mon Sep 17 00:00:00 2001 From: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:20:19 +0200 Subject: [PATCH 07/11] fix: Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource (#4131) * Fix get_table_query_string method for Snowflake datasource Signed-off-by: TomSteenbergen * Add quotes to table string Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen Signed-off-by: Lokesh Rangineni --- sdk/python/feast/infra/offline_stores/snowflake_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index c0b24170996..7ef2dbd6afb 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -191,8 +191,10 @@ def validate(self, config: RepoConfig): def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL.""" - if self.database and self.table: + if self.database and self.schema and self.table: return f'"{self.database}"."{self.schema}"."{self.table}"' + elif self.schema and self.table: + return f'"{self.schema}"."{self.table}"' elif self.table: return f'"{self.table}"' else: From fcde2f13561581b5c41299cc20242a96b85416cf Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Thu, 25 Apr 2024 08:21:06 -0500 Subject: [PATCH 08/11] fix: Change checkout action back to v3 from v5 which isn't released yet (#4147) Signed-off-by: Jeremy Ary Signed-off-by: Lokesh Rangineni --- .github/workflows/master_only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index c355c55c23e..3c244c1b01b 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v3 - name: Setup Python id: setup-python uses: actions/setup-python@v3 From d30914b706cc3a877835cdd54046a6d28af2ae56 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Tue, 23 Apr 2024 16:31:01 -0400 Subject: [PATCH 09/11] changed the code the way mysql container is initialized. Trying to fix the issue - https://github.com/feast-dev/feast/issues/4128 Also going to check if this change will be resolved in the github actions as well. Signed-off-by: Lokesh Rangineni --- sdk/python/tests/unit/test_sql_registry.py | 28 ++++++---------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 094b8967c18..6b5fa3b96d5 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -21,6 +21,7 @@ from pytest_lazyfixture import lazy_fixture from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.mysql import MySqlContainer from feast import FileSource, RequestSource from feast.data_format import ParquetFormat @@ -81,32 +82,17 @@ def pg_registry(): @pytest.fixture(scope="session") def mysql_registry(): - container = ( - DockerContainer("mysql:latest") - .with_exposed_ports(3306) - .with_env("MYSQL_RANDOM_ROOT_PASSWORD", "true") - .with_env("MYSQL_USER", POSTGRES_USER) - .with_env("MYSQL_PASSWORD", POSTGRES_PASSWORD) - .with_env("MYSQL_DATABASE", POSTGRES_DB) - ) - + container = MySqlContainer("mysql:latest") container.start() - # The log string uses '8.0.*' since the version might be changed as new Docker images are pushed. - log_string_to_wait_for = "/usr/sbin/mysqld: ready for connections. Version: '(\\d+(\\.\\d+){1,2})' socket: '/var/run/mysqld/mysqld.sock' port: 3306" # noqa: W605 - waited = wait_for_logs( - container=container, - predicate=log_string_to_wait_for, - timeout=60, - interval=10, - ) - logger.info("Waited for %s seconds until mysql container was up", waited) - container_port = container.get_exposed_port(3306) - container_host = container.get_container_host_ip() + # testing for the database to exist and ready to connect and start testing. + import sqlalchemy + engine = sqlalchemy.create_engine(container.get_connection_url(), pool_pre_ping=True) + engine.connect() registry_config = RegistryConfig( registry_type="sql", - path=f"mysql+pymysql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + path=container.get_connection_url(), sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) From 3fb590c29e9c083df9c4966f297149bb56921dce Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 09:55:00 -0400 Subject: [PATCH 10/11] reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni --- sdk/python/tests/unit/test_sql_registry.py | 123 ++++++++++----------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 6b5fa3b96d5..0e3fb50015c 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -42,7 +42,6 @@ POSTGRES_PASSWORD = "test" POSTGRES_DB = "test" - logger = logging.getLogger(__name__) @@ -145,19 +144,19 @@ def test_apply_entity_success(sql_registry): entity = entities[0] assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" + len(entities) == 1 + and entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" ) entity = sql_registry.get_entity("driver_car_id", project) assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" + entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" ) # After the first apply, the created_timestamp should be the same as the last_update_timestamp. @@ -225,31 +224,31 @@ def test_apply_feature_view_success(sql_registry): # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].features[1].name == "fs1_my_feature_2" - and feature_views[0].features[1].dtype == String - and feature_views[0].features[2].name == "fs1_my_feature_3" - and feature_views[0].features[2].dtype == Array(String) - and feature_views[0].features[3].name == "fs1_my_feature_4" - and feature_views[0].features[3].dtype == Array(Bytes) - and feature_views[0].entities[0] == "fs1_my_entity_1" + len(feature_views) == 1 + and feature_views[0].name == "my_feature_view_1" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].features[1].name == "fs1_my_feature_2" + and feature_views[0].features[1].dtype == String + and feature_views[0].features[2].name == "fs1_my_feature_3" + and feature_views[0].features[2].dtype == Array(String) + and feature_views[0].features[3].name == "fs1_my_feature_4" + and feature_views[0].features[3].dtype == Array(Bytes) + and feature_views[0].entities[0] == "fs1_my_entity_1" ) feature_view = sql_registry.get_feature_view("my_feature_view_1", project) assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.features[1].name == "fs1_my_feature_2" - and feature_view.features[1].dtype == String - and feature_view.features[2].name == "fs1_my_feature_3" - and feature_view.features[2].dtype == Array(String) - and feature_view.features[3].name == "fs1_my_feature_4" - and feature_view.features[3].dtype == Array(Bytes) - and feature_view.entities[0] == "fs1_my_entity_1" + feature_view.name == "my_feature_view_1" + and feature_view.features[0].name == "fs1_my_feature_1" + and feature_view.features[0].dtype == Int64 + and feature_view.features[1].name == "fs1_my_feature_2" + and feature_view.features[1].dtype == String + and feature_view.features[2].name == "fs1_my_feature_3" + and feature_view.features[2].dtype == Array(String) + and feature_view.features[3].name == "fs1_my_feature_4" + and feature_view.features[3].dtype == Array(Bytes) + and feature_view.entities[0] == "fs1_my_entity_1" ) assert feature_view.ttl == timedelta(minutes=5) @@ -339,19 +338,19 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "location_features_from_push" - and feature_views[0].features[0].name == "first_char" - and feature_views[0].features[0].dtype == String + len(feature_views) == 1 + and feature_views[0].name == "location_features_from_push" + and feature_views[0].features[0].name == "first_char" + and feature_views[0].features[0].dtype == String ) feature_view = sql_registry.get_on_demand_feature_view( "location_features_from_push", project ) assert ( - feature_view.name == "location_features_from_push" - and feature_view.features[0].name == "first_char" - and feature_view.features[0].dtype == String + feature_view.name == "location_features_from_push" + and feature_view.features[0].name == "first_char" + and feature_view.features[0].dtype == String ) sql_registry.delete_feature_view("location_features_from_push", project) @@ -441,31 +440,31 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: on_demand_feature_views = sql_registry.list_on_demand_feature_views(project) assert ( - len(on_demand_feature_views) == 1 - and on_demand_feature_views[0].name == "odfv1" - and on_demand_feature_views[0].features[0].name == "odfv1_my_feature_1" - and on_demand_feature_views[0].features[0].dtype == Float32 - and on_demand_feature_views[0].features[1].name == "odfv1_my_feature_2" - and on_demand_feature_views[0].features[1].dtype == Int32 + len(on_demand_feature_views) == 1 + and on_demand_feature_views[0].name == "odfv1" + and on_demand_feature_views[0].features[0].name == "odfv1_my_feature_1" + and on_demand_feature_views[0].features[0].dtype == Float32 + and on_demand_feature_views[0].features[1].name == "odfv1_my_feature_2" + and on_demand_feature_views[0].features[1].dtype == Int32 ) request_schema = on_demand_feature_views[0].get_request_data_schema() assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 + list(request_schema.keys())[0] == "my_input_1" + and list(request_schema.values())[0] == ValueType.INT32 ) feature_view = sql_registry.get_on_demand_feature_view("odfv1", project) assert ( - feature_view.name == "odfv1" - and feature_view.features[0].name == "odfv1_my_feature_1" - and feature_view.features[0].dtype == Float32 - and feature_view.features[1].name == "odfv1_my_feature_2" - and feature_view.features[1].dtype == Int32 + feature_view.name == "odfv1" + and feature_view.features[0].name == "odfv1_my_feature_1" + and feature_view.features[0].dtype == Float32 + and feature_view.features[1].name == "odfv1_my_feature_2" + and feature_view.features[1].dtype == Int32 ) request_schema = feature_view.get_request_data_schema() assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 + list(request_schema.keys())[0] == "my_input_1" + and list(request_schema.values())[0] == ValueType.INT32 ) # Make sure fv1 is untouched @@ -473,19 +472,19 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].entities[0] == "fs1_my_entity_1" + len(feature_views) == 1 + and feature_views[0].name == "my_feature_view_1" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].entities[0] == "fs1_my_entity_1" ) feature_view = sql_registry.get_feature_view("my_feature_view_1", project) assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.entities[0] == "fs1_my_entity_1" + feature_view.name == "my_feature_view_1" + and feature_view.features[0].name == "fs1_my_feature_1" + and feature_view.features[0].dtype == Int64 + and feature_view.entities[0] == "fs1_my_entity_1" ) sql_registry.teardown() From f5ae0bbf9cbf1c4332b1b8b79cd6779d918c8fef Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 10:28:04 -0400 Subject: [PATCH 11/11] reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni --- sdk/python/tests/unit/test_sql_registry.py | 127 +++++++++++---------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 0e3fb50015c..a1460663aeb 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -86,7 +86,10 @@ def mysql_registry(): # testing for the database to exist and ready to connect and start testing. import sqlalchemy - engine = sqlalchemy.create_engine(container.get_connection_url(), pool_pre_ping=True) + + engine = sqlalchemy.create_engine( + container.get_connection_url(), pool_pre_ping=True + ) engine.connect() registry_config = RegistryConfig( @@ -144,19 +147,19 @@ def test_apply_entity_success(sql_registry): entity = entities[0] assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" + len(entities) == 1 + and entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" ) entity = sql_registry.get_entity("driver_car_id", project) assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" + entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" ) # After the first apply, the created_timestamp should be the same as the last_update_timestamp. @@ -224,31 +227,31 @@ def test_apply_feature_view_success(sql_registry): # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].features[1].name == "fs1_my_feature_2" - and feature_views[0].features[1].dtype == String - and feature_views[0].features[2].name == "fs1_my_feature_3" - and feature_views[0].features[2].dtype == Array(String) - and feature_views[0].features[3].name == "fs1_my_feature_4" - and feature_views[0].features[3].dtype == Array(Bytes) - and feature_views[0].entities[0] == "fs1_my_entity_1" + len(feature_views) == 1 + and feature_views[0].name == "my_feature_view_1" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].features[1].name == "fs1_my_feature_2" + and feature_views[0].features[1].dtype == String + and feature_views[0].features[2].name == "fs1_my_feature_3" + and feature_views[0].features[2].dtype == Array(String) + and feature_views[0].features[3].name == "fs1_my_feature_4" + and feature_views[0].features[3].dtype == Array(Bytes) + and feature_views[0].entities[0] == "fs1_my_entity_1" ) feature_view = sql_registry.get_feature_view("my_feature_view_1", project) assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.features[1].name == "fs1_my_feature_2" - and feature_view.features[1].dtype == String - and feature_view.features[2].name == "fs1_my_feature_3" - and feature_view.features[2].dtype == Array(String) - and feature_view.features[3].name == "fs1_my_feature_4" - and feature_view.features[3].dtype == Array(Bytes) - and feature_view.entities[0] == "fs1_my_entity_1" + feature_view.name == "my_feature_view_1" + and feature_view.features[0].name == "fs1_my_feature_1" + and feature_view.features[0].dtype == Int64 + and feature_view.features[1].name == "fs1_my_feature_2" + and feature_view.features[1].dtype == String + and feature_view.features[2].name == "fs1_my_feature_3" + and feature_view.features[2].dtype == Array(String) + and feature_view.features[3].name == "fs1_my_feature_4" + and feature_view.features[3].dtype == Array(Bytes) + and feature_view.entities[0] == "fs1_my_entity_1" ) assert feature_view.ttl == timedelta(minutes=5) @@ -338,19 +341,19 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "location_features_from_push" - and feature_views[0].features[0].name == "first_char" - and feature_views[0].features[0].dtype == String + len(feature_views) == 1 + and feature_views[0].name == "location_features_from_push" + and feature_views[0].features[0].name == "first_char" + and feature_views[0].features[0].dtype == String ) feature_view = sql_registry.get_on_demand_feature_view( "location_features_from_push", project ) assert ( - feature_view.name == "location_features_from_push" - and feature_view.features[0].name == "first_char" - and feature_view.features[0].dtype == String + feature_view.name == "location_features_from_push" + and feature_view.features[0].name == "first_char" + and feature_view.features[0].dtype == String ) sql_registry.delete_feature_view("location_features_from_push", project) @@ -440,31 +443,31 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: on_demand_feature_views = sql_registry.list_on_demand_feature_views(project) assert ( - len(on_demand_feature_views) == 1 - and on_demand_feature_views[0].name == "odfv1" - and on_demand_feature_views[0].features[0].name == "odfv1_my_feature_1" - and on_demand_feature_views[0].features[0].dtype == Float32 - and on_demand_feature_views[0].features[1].name == "odfv1_my_feature_2" - and on_demand_feature_views[0].features[1].dtype == Int32 + len(on_demand_feature_views) == 1 + and on_demand_feature_views[0].name == "odfv1" + and on_demand_feature_views[0].features[0].name == "odfv1_my_feature_1" + and on_demand_feature_views[0].features[0].dtype == Float32 + and on_demand_feature_views[0].features[1].name == "odfv1_my_feature_2" + and on_demand_feature_views[0].features[1].dtype == Int32 ) request_schema = on_demand_feature_views[0].get_request_data_schema() assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 + list(request_schema.keys())[0] == "my_input_1" + and list(request_schema.values())[0] == ValueType.INT32 ) feature_view = sql_registry.get_on_demand_feature_view("odfv1", project) assert ( - feature_view.name == "odfv1" - and feature_view.features[0].name == "odfv1_my_feature_1" - and feature_view.features[0].dtype == Float32 - and feature_view.features[1].name == "odfv1_my_feature_2" - and feature_view.features[1].dtype == Int32 + feature_view.name == "odfv1" + and feature_view.features[0].name == "odfv1_my_feature_1" + and feature_view.features[0].dtype == Float32 + and feature_view.features[1].name == "odfv1_my_feature_2" + and feature_view.features[1].dtype == Int32 ) request_schema = feature_view.get_request_data_schema() assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 + list(request_schema.keys())[0] == "my_input_1" + and list(request_schema.values())[0] == ValueType.INT32 ) # Make sure fv1 is untouched @@ -472,19 +475,19 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: # List Feature Views assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].entities[0] == "fs1_my_entity_1" + len(feature_views) == 1 + and feature_views[0].name == "my_feature_view_1" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].entities[0] == "fs1_my_entity_1" ) feature_view = sql_registry.get_feature_view("my_feature_view_1", project) assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.entities[0] == "fs1_my_entity_1" + feature_view.name == "my_feature_view_1" + and feature_view.features[0].name == "fs1_my_feature_1" + and feature_view.features[0].dtype == Int64 + and feature_view.entities[0] == "fs1_my_entity_1" ) sql_registry.teardown()