From 8f242e640881113f1c6c5137dece8bf01504c61d Mon Sep 17 00:00:00 2001 From: Hai Nguyen Date: Thu, 15 Dec 2022 00:38:18 +0700 Subject: [PATCH 01/10] fix: ODFV able to handle boolean pandas type (#3384) fix: odfv able to handle boolean pandas type Signed-off-by: Hai Nguyen Signed-off-by: Hai Nguyen --- sdk/python/feast/type_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 466993bb3d7..48da7c9acfd 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -147,6 +147,7 @@ def python_type_to_feast_value_type( "uint8": ValueType.INT32, "int8": ValueType.INT32, "bool": ValueType.BOOL, + "boolean": ValueType.BOOL, "timedelta": ValueType.UNIX_TIMESTAMP, "timestamp": ValueType.UNIX_TIMESTAMP, "datetime": ValueType.UNIX_TIMESTAMP, From bc9c07c4b28ff73e067a80bd4379c0867e96876b Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 14:14:02 -0500 Subject: [PATCH 02/10] ci: Fix gcloud CI to have python installed (#3389) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .../fork_workflows/fork_pr_integration_tests_gcp.yml | 2 +- .github/workflows/java_master_only.yml | 10 ++++++++-- .github/workflows/java_pr.yml | 10 ++++++++-- .github/workflows/master_only.yml | 4 ++-- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/pr_integration_tests.yml | 2 +- .github/workflows/publish.yml | 4 ++-- docs/project/maintainers.md | 2 +- 8 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index d53aef01551..2ea859ce640 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -44,7 +44,7 @@ jobs: with: go-version: 1.18.0 - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index c3548991bbf..f5a7fbdb80a 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -21,7 +21,13 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: google-github-actions/setup-gcloud@v0 + - name: Setup Python + uses: actions/setup-python@v2 + id: setup-python + with: + python-version: "3.8" + architecture: x64 + - uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' export_default_credentials: true @@ -150,7 +156,7 @@ jobs: restore-keys: | ${{ runner.os }}-it-maven- - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 72f419e409d..7c5fd37c760 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -72,7 +72,13 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: google-github-actions/setup-gcloud@v0 + - name: Setup Python + uses: actions/setup-python@v2 + id: setup-python + with: + python-version: "3.8" + architecture: x64 + - uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' export_default_credentials: true @@ -122,7 +128,7 @@ jobs: restore-keys: | ${{ runner.os }}-it-maven- - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 13cb341c616..60c3c160c91 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -95,7 +95,7 @@ jobs: with: go-version: ${{ matrix.go-version }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} @@ -195,7 +195,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 8eeb2f0e804..9e4d93202b1 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -145,7 +145,7 @@ jobs: with: go-version: 1.18.0 - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 7c92c293a60..fb60bec691e 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -121,7 +121,7 @@ jobs: with: go-version: 1.18.0 - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 039ab6a24e7..231f532b417 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -65,7 +65,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} @@ -109,7 +109,7 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - uses: actions/checkout@v2 - - uses: google-github-actions/setup-gcloud@v0 + - uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' export_default_credentials: true diff --git a/docs/project/maintainers.md b/docs/project/maintainers.md index ff77c3dfc49..901ce99dc1d 100644 --- a/docs/project/maintainers.md +++ b/docs/project/maintainers.md @@ -36,7 +36,7 @@ Fork specific integration tests are run by the `fork_pr_integration_tests.yml_[p ```yaml - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} From 7b0af0dcffb2f4feda05848757fa40fd0d6c67f4 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 14:34:43 -0500 Subject: [PATCH 03/10] ci: Remove need to download maven cache in CI (#3391) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .github/workflows/java_master_only.yml | 5 ----- .github/workflows/java_pr.yml | 5 ----- .github/workflows/master_only.yml | 5 ----- .github/workflows/publish.yml | 5 ----- infra/scripts/test-java-serving.sh | 4 ---- 5 files changed, 24 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index f5a7fbdb80a..25cb9d2f4fe 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -34,11 +34,6 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - name: Get version run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Build image diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 7c5fd37c760..029cf1c3e81 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -85,11 +85,6 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 60c3c160c91..97a3b603a0c 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -203,11 +203,6 @@ jobs: - name: Use gcloud CLI run: gcloud info - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - name: Build image run: | make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 231f532b417..40aafe63f69 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,11 +73,6 @@ jobs: - name: Use gcloud CLI run: gcloud info - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - name: Build image run: | make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} diff --git a/infra/scripts/test-java-serving.sh b/infra/scripts/test-java-serving.sh index 755722e7a82..7b732beeef3 100755 --- a/infra/scripts/test-java-serving.sh +++ b/infra/scripts/test-java-serving.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ - --output-dir /root/ - mvn -f java/pom.xml --batch-mode --also-make --projects serving test TEST_EXIT_CODE=$? From 62e2005e153800989b080a25310ef90a359963f7 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 16:39:07 -0500 Subject: [PATCH 04/10] ci: Fix workflows to use new gcloud auth step (#3392) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .../fork_pr_integration_tests_gcp.yml | 7 +++++-- .github/workflows/java_master_only.yml | 16 +++++++++++----- .github/workflows/java_pr.yml | 16 +++++++++++----- .github/workflows/master_only.yml | 18 ++++++++++++------ .github/workflows/nightly-ci.yml | 7 +++++-- .github/workflows/pr_integration_tests.yml | 7 +++++-- .github/workflows/publish.yml | 18 ++++++++++++------ docs/project/maintainers.md | 11 +++++++---- 8 files changed, 68 insertions(+), 32 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 2ea859ce640..bab457465c0 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -43,12 +43,15 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Upgrade pip version diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 25cb9d2f4fe..2a2e1433ea5 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -27,12 +27,15 @@ jobs: with: python-version: "3.8" architecture: x64 - - uses: google-github-actions/setup-gcloud@v1 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' - export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - name: Get version run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV @@ -150,12 +153,15 @@ jobs: key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Set up AWS SDK diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 029cf1c3e81..90a1fc2044b 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -78,12 +78,15 @@ jobs: with: python-version: "3.8" architecture: x64 - - uses: google-github-actions/setup-gcloud@v1 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' - export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} @@ -122,12 +125,15 @@ jobs: key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Set up AWS SDK diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 97a3b603a0c..1190ec11e98 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -94,12 +94,15 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} - - name: Set up Cloud SDK + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Set up AWS SDK @@ -194,12 +197,15 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Cloud SDK + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - run: gcloud auth configure-docker --quiet diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 9e4d93202b1..abb4935b752 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -144,12 +144,15 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Set up AWS SDK diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index fb60bec691e..13f2d04437c 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -120,12 +120,15 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - name: Set up AWS SDK diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 40aafe63f69..cb8e459e8c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -64,12 +64,15 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Cloud SDK + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: + version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - name: Use gcloud CLI run: gcloud info - run: gcloud auth configure-docker --quiet @@ -104,12 +107,15 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - uses: actions/checkout@v2 - - uses: google-github-actions/setup-gcloud@v1 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 with: version: '290.0.1' - export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - name: Remove previous Helm run: sudo rm -rf $(which helm) diff --git a/docs/project/maintainers.md b/docs/project/maintainers.md index 901ce99dc1d..b5e01dc4c31 100644 --- a/docs/project/maintainers.md +++ b/docs/project/maintainers.md @@ -35,12 +35,15 @@ Fork specific integration tests are run by the `fork_pr_integration_tests.yml_[p 2. If your offline store/online store needs special setup, add it to the job similar to how gcp is setup. ```yaml - - name: Set up gcloud SDK + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true + version: '290.0.1' + project_id: ${{ secrets.GCP_PROJECT_ID }} ``` 3. Add any environment variables that you need to your github [secrets](https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md). From 8090e2a132114a7354cde195e4dc00d7cb0e88b9 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 16:55:47 -0500 Subject: [PATCH 05/10] ci: Fix workflows to use newer gcloud SDK (#3393) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .github/fork_workflows/fork_pr_integration_tests_gcp.yml | 1 - .github/workflows/java_master_only.yml | 2 -- .github/workflows/java_pr.yml | 2 -- .github/workflows/master_only.yml | 2 -- .github/workflows/nightly-ci.yml | 1 - .github/workflows/pr_integration_tests.yml | 1 - .github/workflows/publish.yml | 2 -- docs/project/maintainers.md | 1 - 8 files changed, 12 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index bab457465c0..4cb22f33fba 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -50,7 +50,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 2a2e1433ea5..356208332fa 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -34,7 +34,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet - name: Get version @@ -160,7 +159,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 90a1fc2044b..ea0ec0d9a9b 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -85,7 +85,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet - name: Build image @@ -132,7 +131,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 1190ec11e98..32c967c6eb3 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -101,7 +101,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info @@ -204,7 +203,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index abb4935b752..2b7eac93018 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -151,7 +151,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 13f2d04437c..9aba7e3bfdc 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -127,7 +127,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb8e459e8c7..11f08bf2e52 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -71,7 +71,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info @@ -114,7 +113,6 @@ jobs: - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet - name: Remove previous Helm diff --git a/docs/project/maintainers.md b/docs/project/maintainers.md index b5e01dc4c31..896401653ce 100644 --- a/docs/project/maintainers.md +++ b/docs/project/maintainers.md @@ -42,7 +42,6 @@ Fork specific integration tests are run by the `fork_pr_integration_tests.yml_[p - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v1 with: - version: '290.0.1' project_id: ${{ secrets.GCP_PROJECT_ID }} ``` From 7b160c74685848a10965d5ab82721eddfb8c3766 Mon Sep 17 00:00:00 2001 From: Ryan Beauchamp Date: Wed, 14 Dec 2022 17:33:22 -0500 Subject: [PATCH 06/10] fix: Remove PySpark dependency from Snowflake Offline Store (#3388) * fix: Wrap pyspark import in try except for snowflake offline store Signed-off-by: Ryan Beauchamp * fix: Capture exception Signed-off-by: Ryan Beauchamp * fix: update jinja template Signed-off-by: Ryan Beauchamp * lint Signed-off-by: Danny Chiao Signed-off-by: Ryan Beauchamp Signed-off-by: Danny Chiao Co-authored-by: Danny Chiao --- infra/templates/README.md.jinja2 | 2 +- sdk/python/feast/infra/offline_stores/snowflake.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index e5213ae9f17..47779d4eb7f 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -42,7 +42,7 @@ pip install feast ### 2. Create a feature repository ```commandline feast init my_feature_repo -cd my_feature_repo +cd my_feature_repo/feature_repo ``` ### 3. Register your feature definitions and set up your feature store diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 330c2ffae54..83aebe7ef9c 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -23,7 +23,6 @@ import pyarrow from pydantic import Field, StrictStr from pydantic.typing import Literal -from pyspark.sql import DataFrame, SparkSession from pytz import utc from feast import OnDemandFeatureView @@ -64,6 +63,13 @@ raise FeastExtrasDependencyImportError("snowflake", str(e)) +try: + from pyspark.sql import DataFrame, SparkSession +except ImportError as e: + from feast.errors import FeastExtrasDependencyImportError + + raise FeastExtrasDependencyImportError("spark", str(e)) + warnings.filterwarnings("ignore", category=DeprecationWarning) From fd97254b18605fff7414845d94725a606112b874 Mon Sep 17 00:00:00 2001 From: "Brehelin.A" Date: Thu, 15 Dec 2022 00:54:04 +0100 Subject: [PATCH 07/10] fix: Get all columns with describe table method from RedshiftData-api (#3377) --- sdk/python/feast/infra/offline_stores/redshift_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/redshift_source.py b/sdk/python/feast/infra/offline_stores/redshift_source.py index d6d51d0310f..4279e6a0687 100644 --- a/sdk/python/feast/infra/offline_stores/redshift_source.py +++ b/sdk/python/feast/infra/offline_stores/redshift_source.py @@ -206,7 +206,8 @@ def get_table_column_names_and_types( client = aws_utils.get_redshift_data_client(config.offline_store.region) if self.table: try: - table = client.describe_table( + paginator = client.get_paginator("describe_table") + response_iterator = paginator.paginate( ClusterIdentifier=config.offline_store.cluster_id, Database=( self.database @@ -217,6 +218,7 @@ def get_table_column_names_and_types( Table=self.table, Schema=self.schema, ) + table = response_iterator.build_full_result() except ClientError as e: if e.response["Error"]["Code"] == "ValidationException": raise RedshiftCredentialsError() from e From 6bcf77c19f84188586ee7dcc57920a43062ee3be Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 22:05:21 -0500 Subject: [PATCH 08/10] fix: Fix bug where SQL registry was incorrectly writing infra config around online stores (#3394) fix: Fix bug where SQL registry was incorrectly writing info around sqlite online store Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- sdk/python/feast/infra/registry/sql.py | 178 +++++++++++---------- sdk/python/tests/unit/test_sql_registry.py | 28 ++++ 2 files changed, 122 insertions(+), 84 deletions(-) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 7867cdff4c3..5e980e90b35 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -207,14 +207,14 @@ def get_stream_feature_view( self, name: str, project: str, allow_cache: bool = False ): return self._get_object( - stream_feature_views, - name, - project, - StreamFeatureViewProto, - StreamFeatureView, - "feature_view_name", - "feature_view_proto", - FeatureViewNotFoundException, + table=stream_feature_views, + name=name, + project=project, + proto_class=StreamFeatureViewProto, + python_class=StreamFeatureView, + id_field_name="feature_view_name", + proto_field_name="feature_view_proto", + not_found_exception=FeatureViewNotFoundException, ) def list_stream_feature_views( @@ -230,101 +230,105 @@ def list_stream_feature_views( def apply_entity(self, entity: Entity, project: str, commit: bool = True): return self._apply_object( - entities, project, "entity_name", entity, "entity_proto" + table=entities, + project=project, + id_field_name="entity_name", + obj=entity, + proto_field_name="entity_proto", ) def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: return self._get_object( - entities, - name, - project, - EntityProto, - Entity, - "entity_name", - "entity_proto", - EntityNotFoundException, + table=entities, + name=name, + project=project, + proto_class=EntityProto, + python_class=Entity, + id_field_name="entity_name", + proto_field_name="entity_proto", + not_found_exception=EntityNotFoundException, ) def get_feature_view( self, name: str, project: str, allow_cache: bool = False ) -> FeatureView: return self._get_object( - feature_views, - name, - project, - FeatureViewProto, - FeatureView, - "feature_view_name", - "feature_view_proto", - FeatureViewNotFoundException, + table=feature_views, + name=name, + project=project, + proto_class=FeatureViewProto, + python_class=FeatureView, + id_field_name="feature_view_name", + proto_field_name="feature_view_proto", + not_found_exception=FeatureViewNotFoundException, ) def get_on_demand_feature_view( self, name: str, project: str, allow_cache: bool = False ) -> OnDemandFeatureView: return self._get_object( - on_demand_feature_views, - name, - project, - OnDemandFeatureViewProto, - OnDemandFeatureView, - "feature_view_name", - "feature_view_proto", - FeatureViewNotFoundException, + table=on_demand_feature_views, + name=name, + project=project, + proto_class=OnDemandFeatureViewProto, + python_class=OnDemandFeatureView, + id_field_name="feature_view_name", + proto_field_name="feature_view_proto", + not_found_exception=FeatureViewNotFoundException, ) def get_request_feature_view(self, name: str, project: str): return self._get_object( - request_feature_views, - name, - project, - RequestFeatureViewProto, - RequestFeatureView, - "feature_view_name", - "feature_view_proto", - FeatureViewNotFoundException, + table=request_feature_views, + name=name, + project=project, + proto_class=RequestFeatureViewProto, + python_class=RequestFeatureView, + id_field_name="feature_view_name", + proto_field_name="feature_view_proto", + not_found_exception=FeatureViewNotFoundException, ) def get_feature_service( self, name: str, project: str, allow_cache: bool = False ) -> FeatureService: return self._get_object( - feature_services, - name, - project, - FeatureServiceProto, - FeatureService, - "feature_service_name", - "feature_service_proto", - FeatureServiceNotFoundException, + table=feature_services, + name=name, + project=project, + proto_class=FeatureServiceProto, + python_class=FeatureService, + id_field_name="feature_service_name", + proto_field_name="feature_service_proto", + not_found_exception=FeatureServiceNotFoundException, ) def get_saved_dataset( self, name: str, project: str, allow_cache: bool = False ) -> SavedDataset: return self._get_object( - saved_datasets, - name, - project, - SavedDatasetProto, - SavedDataset, - "saved_dataset_name", - "saved_dataset_proto", - SavedDatasetNotFound, + table=saved_datasets, + name=name, + project=project, + proto_class=SavedDatasetProto, + python_class=SavedDataset, + id_field_name="saved_dataset_name", + proto_field_name="saved_dataset_proto", + not_found_exception=SavedDatasetNotFound, ) def get_validation_reference( self, name: str, project: str, allow_cache: bool = False ) -> ValidationReference: return self._get_object( - validation_references, - name, - project, - ValidationReferenceProto, - ValidationReference, - "validation_reference_name", - "validation_reference_proto", - ValidationReferenceNotFound, + table=validation_references, + name=name, + project=project, + proto_class=ValidationReferenceProto, + python_class=ValidationReference, + id_field_name="validation_reference_name", + proto_field_name="validation_reference_proto", + not_found_exception=ValidationReferenceNotFound, ) def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: @@ -364,14 +368,14 @@ def get_data_source( self, name: str, project: str, allow_cache: bool = False ) -> DataSource: return self._get_object( - data_sources, - name, - project, - DataSourceProto, - DataSource, - "data_source_name", - "data_source_proto", - DataSourceObjectNotFoundException, + table=data_sources, + name=name, + project=project, + proto_class=DataSourceProto, + python_class=DataSource, + id_field_name="data_source_name", + proto_field_name="data_source_proto", + not_found_exception=DataSourceObjectNotFoundException, ) def list_data_sources( @@ -556,22 +560,28 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr def update_infra(self, infra: Infra, project: str, commit: bool = True): self._apply_object( - managed_infra, project, "infra_name", infra, "infra_proto", name="infra_obj" + table=managed_infra, + project=project, + id_field_name="infra_name", + obj=infra, + proto_field_name="infra_proto", + name="infra_obj", ) def get_infra(self, project: str, allow_cache: bool = False) -> Infra: infra_object = self._get_object( - managed_infra, - "infra_obj", - project, - InfraProto, - Infra, - "infra_name", - "infra_proto", - None, + table=managed_infra, + name="infra_obj", + project=project, + proto_class=InfraProto, + python_class=Infra, + id_field_name="infra_name", + proto_field_name="infra_proto", + not_found_exception=None, ) - infra_object = infra_object or InfraProto() - return Infra.from_proto(infra_object) + if infra_object: + return infra_object + return Infra() def apply_user_metadata( self, diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 877811d0f32..51cb430c9eb 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -28,6 +28,8 @@ from feast.errors import FeatureViewNotFoundException from feast.feature_view import FeatureView from feast.field import Field +from feast.infra.infra_object import Infra +from feast.infra.online_stores.sqlite import SqliteTable from feast.infra.registry.sql import SqlRegistry from feast.on_demand_feature_view import on_demand_feature_view from feast.repo_config import RegistryConfig @@ -258,10 +260,20 @@ def test_apply_feature_view_success(sql_registry): and feature_view.features[3].dtype == Array(Bytes) and feature_view.entities[0] == "fs1_my_entity_1" ) + assert feature_view.ttl == timedelta(minutes=5) # After the first apply, the created_timestamp should be the same as the last_update_timestamp. assert feature_view.created_timestamp == feature_view.last_updated_timestamp + # Modify the feature view and apply again to test if diffing the online store table works + fv1.ttl = timedelta(minutes=6) + sql_registry.apply_feature_view(fv1, project) + feature_views = sql_registry.list_feature_views(project) + assert len(feature_views) == 1 + feature_view = sql_registry.get_feature_view("my_feature_view_1", project) + assert feature_view.ttl == timedelta(minutes=6) + + # Delete feature view sql_registry.delete_feature_view("my_feature_view_1", project) feature_views = sql_registry.list_feature_views(project) assert len(feature_views) == 0 @@ -570,6 +582,22 @@ def test_update_infra(sql_registry): project = "project" infra = sql_registry.get_infra(project=project) + assert len(infra.infra_objects) == 0 + # Should run update infra successfully sql_registry.update_infra(infra, project) + + # Should run update infra successfully when adding + new_infra = Infra() + new_infra.infra_objects.append( + SqliteTable( + path="/tmp/my_path.db", + name="my_table", + ) + ) + sql_registry.update_infra(new_infra, project) + infra = sql_registry.get_infra(project=project) + assert len(infra.infra_objects) == 1 + + # Try again since second time, infra should be not-empty sql_registry.teardown() From 2e573769a24e2429233afe34424af0433b2dc7ec Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 14 Dec 2022 23:01:20 -0500 Subject: [PATCH 09/10] fix: Enable registry caching in SQL Registry (#3395) * fix: Enable registry caching in SQL Registry Signed-off-by: Danny Chiao * docs Signed-off-by: Danny Chiao * fix regular file registry docs too Signed-off-by: Danny Chiao * fix new file lint Signed-off-by: Danny Chiao * fix new file lint Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- docs/getting-started/concepts/registry.md | 20 +- docs/tutorials/using-scalable-registry.md | 1 + .../infra/registry/proto_registry_utils.py | 208 ++++++++++++++++++ sdk/python/feast/infra/registry/registry.py | 169 ++++---------- sdk/python/feast/infra/registry/sql.py | 130 ++++++++++- 5 files changed, 400 insertions(+), 128 deletions(-) create mode 100644 sdk/python/feast/infra/registry/proto_registry_utils.py diff --git a/docs/getting-started/concepts/registry.md b/docs/getting-started/concepts/registry.md index b2280509447..f7d4a5b3e11 100644 --- a/docs/getting-started/concepts/registry.md +++ b/docs/getting-started/concepts/registry.md @@ -16,7 +16,9 @@ a remote file registry, you need to create a GCS / S3 bucket that Feast can unde ```yaml project: feast_demo_aws provider: aws -registry: s3://[YOUR BUCKET YOU CREATED]/registry.pb +registry: + path: s3://[YOUR BUCKET YOU CREATED]/registry.pb + cache_ttl_seconds: 60 online_store: null offline_store: type: file @@ -27,7 +29,9 @@ offline_store: ```yaml project: feast_demo_gcp provider: gcp -registry: gs://[YOUR BUCKET YOU CREATED]/registry.pb +registry: + path: gs://[YOUR BUCKET YOU CREATED]/registry.pb + cache_ttl_seconds: 60 online_store: null offline_store: type: file @@ -43,6 +47,18 @@ multiple feature views or time ranges concurrently). #### SQL Registry Alternatively, a [SQL Registry](../../tutorials/using-scalable-registry.md) can be used for a more scalable registry. +The configuration roughly looks like: +```yaml +project: +provider: +online_store: redis +offline_store: file +registry: + registry_type: sql + path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast + cache_ttl_seconds: 60 +``` + This supports any SQLAlchemy compatible database as a backend. The exact schema can be seen in [sql.py](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py) ### Updating the registry diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md index 0ee02674b13..11d8df47b42 100644 --- a/docs/tutorials/using-scalable-registry.md +++ b/docs/tutorials/using-scalable-registry.md @@ -28,6 +28,7 @@ offline_store: file registry: registry_type: sql path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast + cache_ttl_seconds: 60 ``` Specifically, the registry_type needs to be set to sql in the registry config block. On doing so, the path should refer to the [Database URL](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for the database to be used, as expected by SQLAlchemy. No other additional commands are currently needed to configure this registry. diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py new file mode 100644 index 00000000000..e26b1e10e79 --- /dev/null +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -0,0 +1,208 @@ +from typing import List + +from feast.data_source import DataSource +from feast.entity import Entity +from feast.errors import ( + DataSourceObjectNotFoundException, + EntityNotFoundException, + FeatureServiceNotFoundException, + FeatureViewNotFoundException, + OnDemandFeatureViewNotFoundException, + SavedDatasetNotFound, + ValidationReferenceNotFound, +) +from feast.feature_service import FeatureService +from feast.feature_view import FeatureView +from feast.on_demand_feature_view import OnDemandFeatureView +from feast.project_metadata import ProjectMetadata +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.request_feature_view import RequestFeatureView +from feast.saved_dataset import SavedDataset, ValidationReference +from feast.stream_feature_view import StreamFeatureView + + +def get_feature_service( + registry_proto: RegistryProto, name: str, project: str +) -> FeatureService: + for feature_service_proto in registry_proto.feature_services: + if ( + feature_service_proto.spec.project == project + and feature_service_proto.spec.name == name + ): + return FeatureService.from_proto(feature_service_proto) + raise FeatureServiceNotFoundException(name, project=project) + + +def get_feature_view( + registry_proto: RegistryProto, name: str, project: str +) -> FeatureView: + for feature_view_proto in registry_proto.feature_views: + if ( + feature_view_proto.spec.name == name + and feature_view_proto.spec.project == project + ): + return FeatureView.from_proto(feature_view_proto) + raise FeatureViewNotFoundException(name, project) + + +def get_stream_feature_view( + registry_proto: RegistryProto, name: str, project: str +) -> StreamFeatureView: + for feature_view_proto in registry_proto.stream_feature_views: + if ( + feature_view_proto.spec.name == name + and feature_view_proto.spec.project == project + ): + return StreamFeatureView.from_proto(feature_view_proto) + raise FeatureViewNotFoundException(name, project) + + +def get_request_feature_view(registry_proto: RegistryProto, name: str, project: str): + for feature_view_proto in registry_proto.feature_views: + if ( + feature_view_proto.spec.name == name + and feature_view_proto.spec.project == project + ): + return RequestFeatureView.from_proto(feature_view_proto) + raise FeatureViewNotFoundException(name, project) + + +def get_on_demand_feature_view( + registry_proto: RegistryProto, name: str, project: str +) -> OnDemandFeatureView: + for on_demand_feature_view in registry_proto.on_demand_feature_views: + if ( + on_demand_feature_view.spec.project == project + and on_demand_feature_view.spec.name == name + ): + return OnDemandFeatureView.from_proto(on_demand_feature_view) + raise OnDemandFeatureViewNotFoundException(name, project=project) + + +def get_data_source( + registry_proto: RegistryProto, name: str, project: str +) -> DataSource: + for data_source in registry_proto.data_sources: + if data_source.project == project and data_source.name == name: + return DataSource.from_proto(data_source) + raise DataSourceObjectNotFoundException(name, project=project) + + +def get_entity(registry_proto: RegistryProto, name: str, project: str) -> Entity: + for entity_proto in registry_proto.entities: + if entity_proto.spec.name == name and entity_proto.spec.project == project: + return Entity.from_proto(entity_proto) + raise EntityNotFoundException(name, project=project) + + +def get_saved_dataset( + registry_proto: RegistryProto, name: str, project: str +) -> SavedDataset: + for saved_dataset in registry_proto.saved_datasets: + if saved_dataset.spec.name == name and saved_dataset.spec.project == project: + return SavedDataset.from_proto(saved_dataset) + raise SavedDatasetNotFound(name, project=project) + + +def get_validation_reference( + registry_proto: RegistryProto, name: str, project: str +) -> ValidationReference: + for validation_reference in registry_proto.validation_references: + if ( + validation_reference.name == name + and validation_reference.project == project + ): + return ValidationReference.from_proto(validation_reference) + raise ValidationReferenceNotFound(name, project=project) + + +def list_feature_services( + registry_proto: RegistryProto, project: str, allow_cache: bool = False +) -> List[FeatureService]: + feature_services = [] + for feature_service_proto in registry_proto.feature_services: + if feature_service_proto.spec.project == project: + feature_services.append(FeatureService.from_proto(feature_service_proto)) + return feature_services + + +def list_feature_views( + registry_proto: RegistryProto, project: str +) -> List[FeatureView]: + feature_views: List[FeatureView] = [] + for feature_view_proto in registry_proto.feature_views: + if feature_view_proto.spec.project == project: + feature_views.append(FeatureView.from_proto(feature_view_proto)) + return feature_views + + +def list_request_feature_views( + registry_proto: RegistryProto, project: str +) -> List[RequestFeatureView]: + feature_views: List[RequestFeatureView] = [] + for request_feature_view_proto in registry_proto.request_feature_views: + if request_feature_view_proto.spec.project == project: + feature_views.append( + RequestFeatureView.from_proto(request_feature_view_proto) + ) + return feature_views + + +def list_stream_feature_views( + registry_proto: RegistryProto, project: str +) -> List[StreamFeatureView]: + stream_feature_views = [] + for stream_feature_view in registry_proto.stream_feature_views: + if stream_feature_view.spec.project == project: + stream_feature_views.append( + StreamFeatureView.from_proto(stream_feature_view) + ) + return stream_feature_views + + +def list_on_demand_feature_views( + registry_proto: RegistryProto, project: str +) -> List[OnDemandFeatureView]: + on_demand_feature_views = [] + for on_demand_feature_view in registry_proto.on_demand_feature_views: + if on_demand_feature_view.spec.project == project: + on_demand_feature_views.append( + OnDemandFeatureView.from_proto(on_demand_feature_view) + ) + return on_demand_feature_views + + +def list_entities(registry_proto: RegistryProto, project: str) -> List[Entity]: + entities = [] + for entity_proto in registry_proto.entities: + if entity_proto.spec.project == project: + entities.append(Entity.from_proto(entity_proto)) + return entities + + +def list_data_sources(registry_proto: RegistryProto, project: str) -> List[DataSource]: + data_sources = [] + for data_source_proto in registry_proto.data_sources: + if data_source_proto.project == project: + data_sources.append(DataSource.from_proto(data_source_proto)) + return data_sources + + +def list_saved_datasets( + registry_proto: RegistryProto, project: str, allow_cache: bool = False +) -> List[SavedDataset]: + return [ + SavedDataset.from_proto(saved_dataset) + for saved_dataset in registry_proto.saved_datasets + if saved_dataset.spec.project == project + ] + + +def list_project_metadata( + registry_proto: RegistryProto, project: str +) -> List[ProjectMetadata]: + return [ + ProjectMetadata.from_proto(project_metadata) + for project_metadata in registry_proto.project_metadata + if project_metadata.project == project + ] diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 09d22ee3766..4870c1d45d4 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -30,18 +30,16 @@ from feast.errors import ( ConflictingFeatureViewNames, DataSourceNotFoundException, - DataSourceObjectNotFoundException, EntityNotFoundException, FeatureServiceNotFoundException, FeatureViewNotFoundException, - OnDemandFeatureViewNotFoundException, - SavedDatasetNotFound, ValidationReferenceNotFound, ) from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.importer import import_class from feast.infra.infra_object import Infra +from feast.infra.registry import proto_registry_utils from feast.infra.registry.base_registry import BaseRegistry from feast.infra.registry.registry_store import NoopRegistryStore from feast.on_demand_feature_view import OnDemandFeatureView @@ -293,11 +291,7 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - entities = [] - for entity_proto in registry_proto.entities: - if entity_proto.spec.project == project: - entities.append(Entity.from_proto(entity_proto)) - return entities + return proto_registry_utils.list_entities(registry_proto, project) def list_data_sources( self, project: str, allow_cache: bool = False @@ -305,11 +299,7 @@ def list_data_sources( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - data_sources = [] - for data_source_proto in registry_proto.data_sources: - if data_source_proto.project == project: - data_sources.append(DataSource.from_proto(data_source_proto)) - return data_sources + return proto_registry_utils.list_data_sources(registry_proto, project) def apply_data_source( self, data_source: DataSource, project: str, commit: bool = True @@ -371,36 +361,24 @@ def apply_feature_service( def list_feature_services( self, project: str, allow_cache: bool = False ) -> List[FeatureService]: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - feature_services = [] - for feature_service_proto in registry.feature_services: - if feature_service_proto.spec.project == project: - feature_services.append( - FeatureService.from_proto(feature_service_proto) - ) - return feature_services + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.list_feature_services(registry_proto, project) def get_feature_service( self, name: str, project: str, allow_cache: bool = False ) -> FeatureService: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - - for feature_service_proto in registry.feature_services: - if ( - feature_service_proto.spec.project == project - and feature_service_proto.spec.name == name - ): - return FeatureService.from_proto(feature_service_proto) - raise FeatureServiceNotFoundException(name, project=project) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.get_feature_service(registry_proto, name, project) def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - for entity_proto in registry_proto.entities: - if entity_proto.spec.name == name and entity_proto.spec.project == project: - return Entity.from_proto(entity_proto) - raise EntityNotFoundException(name, project=project) + return proto_registry_utils.get_entity(registry_proto, name, project) def apply_feature_view( self, feature_view: BaseFeatureView, project: str, commit: bool = True @@ -461,49 +439,38 @@ def apply_feature_view( def list_stream_feature_views( self, project: str, allow_cache: bool = False ) -> List[StreamFeatureView]: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - stream_feature_views = [] - for stream_feature_view in registry.stream_feature_views: - if stream_feature_view.spec.project == project: - stream_feature_views.append( - StreamFeatureView.from_proto(stream_feature_view) - ) - return stream_feature_views + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.list_stream_feature_views(registry_proto, project) def list_on_demand_feature_views( self, project: str, allow_cache: bool = False ) -> List[OnDemandFeatureView]: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - on_demand_feature_views = [] - for on_demand_feature_view in registry.on_demand_feature_views: - if on_demand_feature_view.spec.project == project: - on_demand_feature_views.append( - OnDemandFeatureView.from_proto(on_demand_feature_view) - ) - return on_demand_feature_views + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.list_on_demand_feature_views( + registry_proto, project + ) def get_on_demand_feature_view( self, name: str, project: str, allow_cache: bool = False ) -> OnDemandFeatureView: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - - for on_demand_feature_view in registry.on_demand_feature_views: - if ( - on_demand_feature_view.spec.project == project - and on_demand_feature_view.spec.name == name - ): - return OnDemandFeatureView.from_proto(on_demand_feature_view) - raise OnDemandFeatureViewNotFoundException(name, project=project) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.get_on_demand_feature_view( + registry_proto, name, project + ) def get_data_source( self, name: str, project: str, allow_cache: bool = False ) -> DataSource: - registry = self._get_registry_proto(project=project, allow_cache=allow_cache) - - for data_source in registry.data_sources: - if data_source.project == project and data_source.name == name: - return DataSource.from_proto(data_source) - raise DataSourceObjectNotFoundException(name, project=project) + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.get_data_source(registry_proto, name, project) def apply_materialization( self, @@ -570,21 +537,13 @@ def list_feature_views( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - feature_views: List[FeatureView] = [] - for feature_view_proto in registry_proto.feature_views: - if feature_view_proto.spec.project == project: - feature_views.append(FeatureView.from_proto(feature_view_proto)) - return feature_views + return proto_registry_utils.list_feature_views(registry_proto, project) def get_request_feature_view(self, name: str, project: str): registry_proto = self._get_registry_proto(project=project, allow_cache=False) - for feature_view_proto in registry_proto.feature_views: - if ( - feature_view_proto.spec.name == name - and feature_view_proto.spec.project == project - ): - return RequestFeatureView.from_proto(feature_view_proto) - raise FeatureViewNotFoundException(name, project) + return proto_registry_utils.get_request_feature_view( + registry_proto, name, project + ) def list_request_feature_views( self, project: str, allow_cache: bool = False @@ -592,13 +551,7 @@ def list_request_feature_views( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - feature_views: List[RequestFeatureView] = [] - for request_feature_view_proto in registry_proto.request_feature_views: - if request_feature_view_proto.spec.project == project: - feature_views.append( - RequestFeatureView.from_proto(request_feature_view_proto) - ) - return feature_views + return proto_registry_utils.list_request_feature_views(registry_proto, project) def get_feature_view( self, name: str, project: str, allow_cache: bool = False @@ -606,13 +559,7 @@ def get_feature_view( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - for feature_view_proto in registry_proto.feature_views: - if ( - feature_view_proto.spec.name == name - and feature_view_proto.spec.project == project - ): - return FeatureView.from_proto(feature_view_proto) - raise FeatureViewNotFoundException(name, project) + return proto_registry_utils.get_feature_view(registry_proto, name, project) def get_stream_feature_view( self, name: str, project: str, allow_cache: bool = False @@ -620,13 +567,9 @@ def get_stream_feature_view( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - for feature_view_proto in registry_proto.stream_feature_views: - if ( - feature_view_proto.spec.name == name - and feature_view_proto.spec.project == project - ): - return StreamFeatureView.from_proto(feature_view_proto) - raise FeatureViewNotFoundException(name, project) + return proto_registry_utils.get_stream_feature_view( + registry_proto, name, project + ) def delete_feature_service(self, name: str, project: str, commit: bool = True): self._prepare_registry_for_changes(project) @@ -753,13 +696,7 @@ def get_saved_dataset( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - for saved_dataset in registry_proto.saved_datasets: - if ( - saved_dataset.spec.name == name - and saved_dataset.spec.project == project - ): - return SavedDataset.from_proto(saved_dataset) - raise SavedDatasetNotFound(name, project=project) + return proto_registry_utils.get_saved_dataset(registry_proto, name, project) def list_saved_datasets( self, project: str, allow_cache: bool = False @@ -767,11 +704,7 @@ def list_saved_datasets( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return [ - SavedDataset.from_proto(saved_dataset) - for saved_dataset in registry_proto.saved_datasets - if saved_dataset.spec.project == project - ] + return proto_registry_utils.list_saved_datasets(registry_proto, project) def apply_validation_reference( self, @@ -803,13 +736,9 @@ def get_validation_reference( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - for validation_reference in registry_proto.validation_references: - if ( - validation_reference.name == name - and validation_reference.project == project - ): - return ValidationReference.from_proto(validation_reference) - raise ValidationReferenceNotFound(name, project=project) + return proto_registry_utils.get_validation_reference( + registry_proto, name, project + ) def delete_validation_reference(self, name: str, project: str, commit: bool = True): registry_proto = self._prepare_registry_for_changes(project) @@ -832,11 +761,7 @@ def list_project_metadata( registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return [ - ProjectMetadata.from_proto(project_metadata) - for project_metadata in registry_proto.project_metadata - if project_metadata.project == project - ] + return proto_registry_utils.list_project_metadata(registry_proto, project) def commit(self): """Commits the state of the registry cache to the remote registry store.""" diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 5e980e90b35..f0782515443 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -1,7 +1,8 @@ import uuid -from datetime import datetime +from datetime import datetime, timedelta from enum import Enum from pathlib import Path +from threading import Lock from typing import Any, Callable, List, Optional, Set, Union from sqlalchemy import ( # type: ignore @@ -34,6 +35,7 @@ from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.infra.infra_object import Infra +from feast.infra.registry import proto_registry_utils from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView from feast.project_metadata import ProjectMetadata @@ -183,6 +185,14 @@ def __init__( assert registry_config is not None, "SqlRegistry needs a valid registry_config" self.engine: Engine = create_engine(registry_config.path, echo=False) metadata.create_all(self.engine) + self.cached_registry_proto = self.proto() + self.cached_registry_proto_created = datetime.utcnow() + self._refresh_lock = Lock() + self.cached_registry_proto_ttl = timedelta( + seconds=registry_config.cache_ttl_seconds + if registry_config.cache_ttl_seconds is not None + else 0 + ) def teardown(self): for t in { @@ -200,12 +210,37 @@ def teardown(self): conn.execute(stmt) def refresh(self, project: Optional[str]): - # This method is a no-op since we're always reading the latest values from the db. - pass + self.cached_registry_proto = self.proto() + self.cached_registry_proto_created = datetime.utcnow() + + def _refresh_cached_registry_if_necessary(self): + with self._refresh_lock: + expired = ( + self.cached_registry_proto is None + or self.cached_registry_proto_created is None + ) or ( + self.cached_registry_proto_ttl.total_seconds() + > 0 # 0 ttl means infinity + and ( + datetime.utcnow() + > ( + self.cached_registry_proto_created + + self.cached_registry_proto_ttl + ) + ) + ) + + if expired: + self.refresh() def get_stream_feature_view( self, name: str, project: str, allow_cache: bool = False ): + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_stream_feature_view( + self.cached_registry_proto, name, project + ) return self._get_object( table=stream_feature_views, name=name, @@ -220,6 +255,11 @@ def get_stream_feature_view( def list_stream_feature_views( self, project: str, allow_cache: bool = False ) -> List[StreamFeatureView]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_stream_feature_views( + self.cached_registry_proto, project + ) return self._list_objects( stream_feature_views, project, @@ -238,6 +278,11 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): ) def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_entity( + self.cached_registry_proto, name, project + ) return self._get_object( table=entities, name=name, @@ -252,6 +297,11 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti def get_feature_view( self, name: str, project: str, allow_cache: bool = False ) -> FeatureView: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_feature_view( + self.cached_registry_proto, name, project + ) return self._get_object( table=feature_views, name=name, @@ -266,6 +316,11 @@ def get_feature_view( def get_on_demand_feature_view( self, name: str, project: str, allow_cache: bool = False ) -> OnDemandFeatureView: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_on_demand_feature_view( + self.cached_registry_proto, name, project + ) return self._get_object( table=on_demand_feature_views, name=name, @@ -277,7 +332,14 @@ def get_on_demand_feature_view( not_found_exception=FeatureViewNotFoundException, ) - def get_request_feature_view(self, name: str, project: str): + def get_request_feature_view( + self, name: str, project: str, allow_cache: bool = False + ): + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_request_feature_view( + self.cached_registry_proto, name, project + ) return self._get_object( table=request_feature_views, name=name, @@ -292,6 +354,11 @@ def get_request_feature_view(self, name: str, project: str): def get_feature_service( self, name: str, project: str, allow_cache: bool = False ) -> FeatureService: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_feature_service( + self.cached_registry_proto, name, project + ) return self._get_object( table=feature_services, name=name, @@ -306,6 +373,11 @@ def get_feature_service( def get_saved_dataset( self, name: str, project: str, allow_cache: bool = False ) -> SavedDataset: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_saved_dataset( + self.cached_registry_proto, name, project + ) return self._get_object( table=saved_datasets, name=name, @@ -320,6 +392,11 @@ def get_saved_dataset( def get_validation_reference( self, name: str, project: str, allow_cache: bool = False ) -> ValidationReference: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_validation_reference( + self.cached_registry_proto, name, project + ) return self._get_object( table=validation_references, name=name, @@ -332,6 +409,11 @@ def get_validation_reference( ) def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_entities( + self.cached_registry_proto, project + ) return self._list_objects( entities, project, EntityProto, Entity, "entity_proto" ) @@ -367,6 +449,11 @@ def delete_feature_service(self, name: str, project: str, commit: bool = True): def get_data_source( self, name: str, project: str, allow_cache: bool = False ) -> DataSource: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.get_data_source( + self.cached_registry_proto, name, project + ) return self._get_object( table=data_sources, name=name, @@ -381,6 +468,11 @@ def get_data_source( def list_data_sources( self, project: str, allow_cache: bool = False ) -> List[DataSource]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_data_sources( + self.cached_registry_proto, project + ) return self._list_objects( data_sources, project, DataSourceProto, DataSource, "data_source_proto" ) @@ -425,6 +517,11 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): def list_feature_services( self, project: str, allow_cache: bool = False ) -> List[FeatureService]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_feature_services( + self.cached_registry_proto, project + ) return self._list_objects( feature_services, project, @@ -436,6 +533,11 @@ def list_feature_services( def list_feature_views( self, project: str, allow_cache: bool = False ) -> List[FeatureView]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_feature_views( + self.cached_registry_proto, project + ) return self._list_objects( feature_views, project, FeatureViewProto, FeatureView, "feature_view_proto" ) @@ -443,6 +545,11 @@ def list_feature_views( def list_saved_datasets( self, project: str, allow_cache: bool = False ) -> List[SavedDataset]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_saved_datasets( + self.cached_registry_proto, project + ) return self._list_objects( saved_datasets, project, @@ -454,6 +561,11 @@ def list_saved_datasets( def list_request_feature_views( self, project: str, allow_cache: bool = False ) -> List[RequestFeatureView]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_request_feature_views( + self.cached_registry_proto, project + ) return self._list_objects( request_feature_views, project, @@ -465,6 +577,11 @@ def list_request_feature_views( def list_on_demand_feature_views( self, project: str, allow_cache: bool = False ) -> List[OnDemandFeatureView]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_on_demand_feature_views( + self.cached_registry_proto, project + ) return self._list_objects( on_demand_feature_views, project, @@ -476,6 +593,11 @@ def list_on_demand_feature_views( def list_project_metadata( self, project: str, allow_cache: bool = False ) -> List[ProjectMetadata]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_project_metadata( + self.cached_registry_proto, project + ) with self.engine.connect() as conn: stmt = select(feast_metadata).where( feast_metadata.c.project_id == project, From cb163fc14562b47941af72ef18ebd3747a721c5d Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Thu, 15 Dec 2022 04:20:20 +0000 Subject: [PATCH 10/10] chore(release): release 0.27.1 ## [0.27.1](https://github.com/feast-dev/feast/compare/v0.27.0...v0.27.1) (2022-12-15) ### Bug Fixes * Enable registry caching in SQL Registry ([#3395](https://github.com/feast-dev/feast/issues/3395)) ([2e57376](https://github.com/feast-dev/feast/commit/2e573769a24e2429233afe34424af0433b2dc7ec)) * Fix bug where SQL registry was incorrectly writing infra config around online stores ([#3394](https://github.com/feast-dev/feast/issues/3394)) ([6bcf77c](https://github.com/feast-dev/feast/commit/6bcf77c19f84188586ee7dcc57920a43062ee3be)) * Get all columns with describe table method from RedshiftData-api ([#3377](https://github.com/feast-dev/feast/issues/3377)) ([fd97254](https://github.com/feast-dev/feast/commit/fd97254b18605fff7414845d94725a606112b874)) * ODFV able to handle boolean pandas type ([#3384](https://github.com/feast-dev/feast/issues/3384)) ([8f242e6](https://github.com/feast-dev/feast/commit/8f242e640881113f1c6c5137dece8bf01504c61d)) * Remove PySpark dependency from Snowflake Offline Store ([#3388](https://github.com/feast-dev/feast/issues/3388)) ([7b160c7](https://github.com/feast-dev/feast/commit/7b160c74685848a10965d5ab82721eddfb8c3766)) --- CHANGELOG.md | 11 +++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 ++-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- infra/charts/feast/charts/feature-server/values.yaml | 2 +- .../feast/charts/transformation-service/Chart.yaml | 4 ++-- .../feast/charts/transformation-service/README.md | 4 ++-- .../feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 17 files changed, 38 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b2f585587a..81b1625a9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.27.1](https://github.com/feast-dev/feast/compare/v0.27.0...v0.27.1) (2022-12-15) + + +### Bug Fixes + +* Enable registry caching in SQL Registry ([#3395](https://github.com/feast-dev/feast/issues/3395)) ([2e57376](https://github.com/feast-dev/feast/commit/2e573769a24e2429233afe34424af0433b2dc7ec)) +* Fix bug where SQL registry was incorrectly writing infra config around online stores ([#3394](https://github.com/feast-dev/feast/issues/3394)) ([6bcf77c](https://github.com/feast-dev/feast/commit/6bcf77c19f84188586ee7dcc57920a43062ee3be)) +* Get all columns with describe table method from RedshiftData-api ([#3377](https://github.com/feast-dev/feast/issues/3377)) ([fd97254](https://github.com/feast-dev/feast/commit/fd97254b18605fff7414845d94725a606112b874)) +* ODFV able to handle boolean pandas type ([#3384](https://github.com/feast-dev/feast/issues/3384)) ([8f242e6](https://github.com/feast-dev/feast/commit/8f242e640881113f1c6c5137dece8bf01504c61d)) +* Remove PySpark dependency from Snowflake Offline Store ([#3388](https://github.com/feast-dev/feast/issues/3388)) ([7b160c7](https://github.com/feast-dev/feast/commit/7b160c74685848a10965d5ab82721eddfb8c3766)) + # [0.27.0](https://github.com/feast-dev/feast/compare/v0.26.0...v0.27.0) (2022-12-05) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 55041c2b8dd..b224ce67181 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.27.0 +version: 0.27.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index f023021f963..51c63793f9b 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.27.0` +Current chart version is `0.27.1` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.27.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.27.1"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index c66c45e36dc..b4def159ff7 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.27.0 + tag: 0.27.1 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 2586871119d..a6a96b17aa5 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.27.0 +version: 0.27.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 9519729f541..fd21623f908 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.27.0` +Feature store for machine learning Current chart version is `0.27.1` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.27.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.27.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.27.1 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.27.1 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 4874dca14ba..bdca01b0b6d 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.27.0 -appVersion: v0.27.0 +version: 0.27.1 +appVersion: v0.27.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 92dd950a22a..93785d26d25 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.27.0](https://img.shields.io/badge/Version-0.27.0-informational?style=flat-square) ![AppVersion: v0.27.0](https://img.shields.io/badge/AppVersion-v0.27.0-informational?style=flat-square) +![Version: 0.27.1](https://img.shields.io/badge/Version-0.27.1-informational?style=flat-square) ![AppVersion: v0.27.1](https://img.shields.io/badge/AppVersion-v0.27.1-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.27.0"` | Image tag | +| image.tag | string | `"0.27.1"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 529ca53d09a..c8a9269ebde 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.27.0 + tag: 0.27.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index cf12aa7a79a..96e9f26928e 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.27.0 -appVersion: v0.27.0 +version: 0.27.1 +appVersion: v0.27.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index b137e9b544b..1dde954567e 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.27.0](https://img.shields.io/badge/Version-0.27.0-informational?style=flat-square) ![AppVersion: v0.27.0](https://img.shields.io/badge/AppVersion-v0.27.0-informational?style=flat-square) +![Version: 0.27.1](https://img.shields.io/badge/Version-0.27.1-informational?style=flat-square) ![AppVersion: v0.27.1](https://img.shields.io/badge/AppVersion-v0.27.1-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.27.0"` | Image tag | +| image.tag | string | `"0.27.1"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index eb5876a18b3..a92b4a4d58f 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.27.0 + tag: 0.27.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 94d34d0f335..203818fe082 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.27.0 + version: 0.27.1 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.27.0 + version: 0.27.1 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 5f1c8f5df90..061cc44ad7b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.27.0 + 0.27.1 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 259d48ea50a..3af70924ff0 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.27.0", + "@feast-dev/feast-ui": "0.27.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index c168c310f69..037f7e0e561 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1300,10 +1300,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.27.0": - version "0.27.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.27.0.tgz#3a0c2ae54e0a0cfb94a6d9883d91d3dd5e649b2c" - integrity sha512-1meDDMt0SNR6u45zH9wVgakrejrywSlOPJnq5JLLkErOVLmJsFe1EN6oeGdJXKQKuk5rQGgwWX0y7Ox4o2T45g== +"@feast-dev/feast-ui@0.27.1": + version "0.27.1" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.27.1.tgz#e881fd59e6a833568cdbaebe71209eafa1bd77c5" + integrity sha512-DP7syQ70uBK99qq2fS/+uhekj17O6wYvCqNrqkI1Ddp41+MiX1AXMagJuEfK04SqOvlUSm9DokAd7sgKpU/IRg== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index 948c1655ba9..8ef620b6a60 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.27.0", + "version": "0.27.1", "private": false, "files": [ "dist"