From fb0fb2b0febb780b567e4908d6e92369035b14b8 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 5 Dec 2024 15:19:48 -0500 Subject: [PATCH 01/90] chore: Updating Upload Artifacts to v4 (#4819) Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 4 ++-- .github/workflows/java_master_only.yml | 2 +- .github/workflows/java_pr.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8e52ba12c9e..c0c25edf223 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -72,7 +72,7 @@ jobs: run: | python -m pip install build python -m build --wheel --outdir wheelhouse/ - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheels path: ./wheelhouse/*.whl @@ -104,7 +104,7 @@ jobs: - name: Build run: | python3 setup.py sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheels path: dist/* diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 127b59c5437..2475321f706 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -86,7 +86,7 @@ jobs: ${{ runner.os }}-ut-maven- - name: Test java run: make test-java-with-coverage - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-coverage-report path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 8b83646c11f..0391e6fde9f 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -67,7 +67,7 @@ jobs: ${{ runner.os }}-ut-maven- - name: Test java run: make test-java-with-coverage - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-coverage-report path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ @@ -184,7 +184,7 @@ jobs: - name: Run integration tests run: make test-java-integration - name: Save report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: it-report From e0b5d0c87407c50cacdf03eded8e4b52c9379e3b Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 5 Dec 2024 15:37:43 -0500 Subject: [PATCH 02/90] chore: Update publish to allow for manual trigger (#4820) * chore: Update publish to allow manual trigger Signed-off-by: Francisco Javier Arceo * chore: Updating publish to try and manually trigger it Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7d5dca8e08b..8566e4920ed 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,11 +4,27 @@ on: push: tags: - 'v*.*.*' + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string jobs: get-version: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GIT_AUTHOR_NAME: feast-ci-bot + GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co + GIT_COMMITTER_NAME: feast-ci-bot + GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co outputs: release_version: ${{ steps.get_release_version.outputs.release_version }} version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} @@ -17,7 +33,13 @@ jobs: - uses: actions/checkout@v4 - name: Get release version id: get_release_version - run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + echo "Using custom version: ${{ github.event.inputs.custom_version }}" + echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" + else + echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + fi - name: Get release version without prefix id: get_release_version_without_prefix env: From 3fbff08d853cf8309c9bb8262f1cf0e3d4e4f89c Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 15:52:38 -0500 Subject: [PATCH 03/90] chore: Updating collision from artifact names Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c0c25edf223..c3f0dd519a4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -74,7 +74,7 @@ jobs: python -m build --wheel --outdir wheelhouse/ - uses: actions/upload-artifact@v4 with: - name: wheels + name: python-wheels path: ./wheelhouse/*.whl build-source-distribution: @@ -106,7 +106,7 @@ jobs: python3 setup.py sdist - uses: actions/upload-artifact@v4 with: - name: wheels + name: source-distribution path: dist/* # We add this step so the docker images can be built as part of the pre-release verification steps. @@ -161,7 +161,11 @@ jobs: architecture: x64 - uses: actions/download-artifact@v4.1.7 with: - name: wheels + name: python-wheels + path: dist + - uses: actions/download-artifact@v4.1.7 + with: + name: source-distribution path: dist - name: Install OS X dependencies if: matrix.os == 'macos-13' From 8e4b09bba56ceea54b623127076292dbd8f682c1 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 16:20:24 -0500 Subject: [PATCH 04/90] chore: Adjusting the semantic version parsing in the publish Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8566e4920ed..85da150dddf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,21 +31,40 @@ jobs: highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: - uses: actions/checkout@v4 + - name: Validate custom version input + id: validate_custom_version + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then + echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." + exit 1 + fi + echo "Validated custom version: ${{ github.event.inputs.custom_version }}" + fi - name: Get release version id: get_release_version run: | if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then echo "Using custom version: ${{ github.event.inputs.custom_version }}" echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" + elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then + echo "Using tag reference: ${GITHUB_REF#refs/tags/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" else - echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" fi - name: Get release version without prefix id: get_release_version_without_prefix env: RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} run: | - echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} + if [[ "${RELEASE_VERSION}" == v* ]]; then + echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" + else + echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" + fi - name: Get highest semver id: get_highest_semver env: From 364955499bd72bd544b4cea685f9a8843382598a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 16:48:33 -0500 Subject: [PATCH 05/90] chore: Updating build_wheels to accept input from publish workflow Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 14 +++++++++++++- .github/workflows/publish.yml | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c3f0dd519a4..7b2a3e40551 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -8,6 +8,11 @@ on: tags: - 'v*.*.*' workflow_call: + inputs: + release_version: + description: 'The release version to use (e.g., v1.2.3)' + required: true + type: string jobs: get-version: @@ -23,7 +28,14 @@ jobs: persist-credentials: false - name: Get release version id: get_release_version - run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + run: | + if [[ -n "${{ inputs.release_version }}" ]]; then + echo "Using provided release version: ${{ inputs.release_version }}" + echo "::set-output name=release_version::${{ inputs.release_version }}" + else + echo "No release version provided. Falling back to GITHUB_REF." + echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" + fi - name: Get release version without prefix id: get_release_version_without_prefix env: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 85da150dddf..bb6f75b3bfa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -42,6 +42,7 @@ jobs: fi echo "Validated custom version: ${{ github.event.inputs.custom_version }}" fi + - name: Get release version id: get_release_version run: | @@ -169,6 +170,8 @@ jobs: build_wheels: uses: ./.github/workflows/build_wheels.yml + with: + release_version: ${{ github.event.inputs.custom_version }} publish-python-sdk: if: github.repository == 'feast-dev/feast' From 19424bcc975d90d922791b5bd0da6ac13955c0c5 Mon Sep 17 00:00:00 2001 From: Abdul Hameed Date: Thu, 5 Dec 2024 17:19:10 -0500 Subject: [PATCH 06/90] feat: Feast Operator support log level configuration for services (#4808) * fix: Feast Operator updated the kustomize version to v5.5.0 Signed-off-by: Abdul Hameed * feat : Feast Operator support log level configuration for services Signed-off-by: Abdul Hameed * Update infra/feast-operator/internal/controller/services/services.go Co-authored-by: Tommy Hughes IV Signed-off-by: Abdul Hameed * Update infra/feast-operator/internal/controller/services/services.go Co-authored-by: Tommy Hughes IV Signed-off-by: Abdul Hameed * added unit test for loglevel Signed-off-by: Abdul Hameed * fix the loglevel command Signed-off-by: Abdul Hameed * moved the logLevel under LocalRegistryConfig and updated testcase to validate it Signed-off-by: Abdul Hameed --------- Signed-off-by: Abdul Hameed Co-authored-by: Tommy Hughes IV --- .../api/v1alpha1/featurestore_types.go | 12 + .../crd/bases/feast.dev_featurestores.yaml | 66 +++++ ...alpha1_featurestore_services_loglevel.yaml | 15 + infra/feast-operator/dist/install.yaml | 68 ++++- .../featurestore_controller_loglevel_test.go | 262 ++++++++++++++++++ .../internal/controller/services/services.go | 38 ++- .../controller/services/services_types.go | 8 +- 7 files changed, 460 insertions(+), 9 deletions(-) create mode 100644 infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml create mode 100644 infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 17a029c02ea..0b516630cd2 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -77,6 +77,10 @@ type OfflineStore struct { ServiceConfigs `json:",inline"` Persistence *OfflineStorePersistence `json:"persistence,omitempty"` TLS *OfflineTlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the offline store service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // OfflineTlsConfigs configures server TLS for the offline feast service. in an openshift cluster, this is configured by default using service serving certificates. @@ -130,6 +134,10 @@ type OnlineStore struct { ServiceConfigs `json:",inline"` Persistence *OnlineStorePersistence `json:"persistence,omitempty"` TLS *TlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the online store service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // OnlineStorePersistence configures the persistence settings for the online store service @@ -177,6 +185,10 @@ type LocalRegistryConfig struct { ServiceConfigs `json:",inline"` Persistence *RegistryPersistence `json:"persistence,omitempty"` TLS *TlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the registry service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // RegistryPersistence configures the persistence settings for the registry service diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 1402a64056c..6929796d34f 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -227,6 +227,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -576,6 +587,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -939,6 +961,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -1429,6 +1462,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -1784,6 +1828,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2155,6 +2210,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml new file mode 100644 index 00000000000..7ae96d44f01 --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml @@ -0,0 +1,15 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-services-loglevel +spec: + feastProject: my_project + services: + onlineStore: + logLevel: debug + offlineStore: + logLevel: debug + registry: + local: + logLevel: info + diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 18ab82e9ca2..9e213994eba 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -235,6 +235,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -584,6 +595,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -947,6 +969,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -1437,6 +1470,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -1792,6 +1836,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2163,6 +2218,17 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -2894,7 +2960,7 @@ spec: - --leader-elect command: - /manager - image: feastdev/feast-operator:0.41.0 + image: feastdev/feast-operator:0.42.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go new file mode 100644 index 00000000000..70f33486fce --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { + Context("When reconciling a FeatureStore resource", func() { + const resourceName = "test-loglevel" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + featurestore := &feastdevv1alpha1.FeatureStore{} + + BeforeEach(func() { + By("creating the custom resource for the Kind FeatureStore") + err := k8sClient.Get(ctx, typeNamespacedName, featurestore) + if err != nil && errors.IsNotFound(err) { + resource := &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{ + LogLevel: "error", + }, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{ + LogLevel: "debug", + }, + OfflineStore: &feastdevv1alpha1.OfflineStore{ + LogLevel: "info", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + } + }) + AfterEach(func() { + resource := &feastdevv1alpha1.FeatureStore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + By("Cleanup the specific resource instance FeatureStore") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + + It("should successfully reconcile the resource with logLevel", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + feast := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + }, + } + + Expect(resource.Status).NotTo(BeNil()) + Expect(resource.Status.FeastVersion).To(Equal(feastversion.FeastVersion)) + Expect(resource.Status.Applied.FeastProject).To(Equal(resource.Spec.FeastProject)) + Expect(resource.Status.Applied.Services).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry).NotTo(BeNil()) + + Expect(resource.Status.Conditions).NotTo(BeEmpty()) + cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.RegistryReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ClientReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ClientReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OfflineStoreReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OnlineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) + Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("ERROR")) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command = deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("INFO")) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command = deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("DEBUG")) + }) + + It("should not include --log-level parameter when logLevel is not specified for any service", func() { + By("Updating the FeatureStore resource without specifying logLevel for any service") + resource := &feastdevv1alpha1.FeatureStore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + resource.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{}, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + } + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + feast := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + }, + } + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).NotTo(ContainElement("--log-level")) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command = deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).NotTo(ContainElement("--log-level")) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + command = deploy.Spec.Template.Spec.Containers[0].Command + Expect(command).NotTo(ContainElement("--log-level")) + }) + + }) +}) diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index b1878ee00ae..60aabebe024 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -355,25 +355,36 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F } func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { + baseCommand := "feast" + options := []string{} + logLevel := feast.getLogLevelForType(feastType) + if logLevel != nil { + options = append(options, "--log-level", strings.ToUpper(*logLevel)) + } + deploySettings := FeastServiceConstants[feastType] targetPort := deploySettings.TargetHttpPort tls := feast.getTlsConfigs(feastType) if tls.IsTLS() { targetPort = deploySettings.TargetHttpsPort feastTlsPath := GetTlsPath(feastType) - deploySettings.Command = append(deploySettings.Command, []string{"--key", feastTlsPath + tls.SecretKeyNames.TlsKey, + deploySettings.Args = append(deploySettings.Args, []string{"--key", feastTlsPath + tls.SecretKeyNames.TlsKey, "--cert", feastTlsPath + tls.SecretKeyNames.TlsCrt}...) } - deploySettings.Command = append(deploySettings.Command, []string{"-p", strconv.Itoa(int(targetPort))}...) + deploySettings.Args = append(deploySettings.Args, []string{"-p", strconv.Itoa(int(targetPort))}...) if feastType == OfflineFeastType { if tls.IsTLS() && feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient != nil { - deploySettings.Command = append(deploySettings.Command, + deploySettings.Args = append(deploySettings.Args, []string{"--verify_client", strconv.FormatBool(*feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient)}...) } } - return deploySettings.Command + // Combine base command, options, and arguments + feastCommand := append([]string{baseCommand}, options...) + feastCommand = append(feastCommand, deploySettings.Args...) + + return feastCommand } func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { @@ -474,6 +485,25 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } +func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { + services := feast.Handler.FeatureStore.Status.Applied.Services + switch feastType { + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.LogLevel != "" { + return &services.OfflineStore.LogLevel + } + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.LogLevel != "" { + return &services.OnlineStore.LogLevel + } + case RegistryFeastType: + if services.Registry != nil && services.Registry.Local.LogLevel != "" { + return &services.Registry.Local.LogLevel + } + } + return nil +} + // GetObjectMeta returns the feast k8s object metadata func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.Handler.FeatureStore.Namespace} diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 2c454459d88..b7c0f5f048b 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -87,17 +87,17 @@ var ( FeastServiceConstants = map[FeastServiceType]deploymentSettings{ OfflineFeastType: { - Command: []string{"feast", "serve_offline", "-h", "0.0.0.0"}, + Args: []string{"serve_offline", "-h", "0.0.0.0"}, TargetHttpPort: 8815, TargetHttpsPort: 8816, }, OnlineFeastType: { - Command: []string{"feast", "serve", "-h", "0.0.0.0"}, + Args: []string{"serve", "-h", "0.0.0.0"}, TargetHttpPort: 6566, TargetHttpsPort: 6567, }, RegistryFeastType: { - Command: []string{"feast", "serve_registry"}, + Args: []string{"serve_registry"}, TargetHttpPort: 6570, TargetHttpsPort: 6571, }, @@ -234,7 +234,7 @@ type AuthzConfig struct { } type deploymentSettings struct { - Command []string + Args []string TargetHttpPort int32 TargetHttpsPort int32 } From 5ce025ff73c300ec1d07e4f9cd6a2b1deac7c2a3 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 17:02:57 -0500 Subject: [PATCH 07/90] chore: Adding semantic version as input to build wheels as well Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bb6f75b3bfa..a3d85f598e9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,6 +16,10 @@ on: type: string jobs: + prepare-versions: + runs-on: ubuntu-latest + outputs: + highest_semver_tag: ${{ steps.strip_prefix.outputs.highest_semver_tag }} get-version: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest @@ -42,6 +46,13 @@ jobs: fi echo "Validated custom version: ${{ github.event.inputs.custom_version }}" fi + - name: Strip 'v' Prefix for Highest SemVer Tag + id: strip_prefix + run: | + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version#v }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Highest SemVer Tag: $HIGHEST_SEMVER_TAG" + - name: Get release version id: get_release_version @@ -172,6 +183,7 @@ jobs: uses: ./.github/workflows/build_wheels.yml with: release_version: ${{ github.event.inputs.custom_version }} + highest_semver_tag: ${{ needs.prepare-versions.outputs.highest_semver_tag }} publish-python-sdk: if: github.repository == 'feast-dev/feast' From a2fc19145a4782de7207a090589cdfb2e75aacfd Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 18:29:01 -0500 Subject: [PATCH 08/90] adding input Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7b2a3e40551..d23fd23325e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,6 +13,10 @@ on: description: 'The release version to use (e.g., v1.2.3)' required: true type: string + highest_semver_tag: + description: 'The highest semantic version tag without the "v" prefix (e.g., 1.2.3)' + required: true + type: string jobs: get-version: From e3e8c975b4b9891913d0be8d50df909d4d243191 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 19:00:09 -0500 Subject: [PATCH 09/90] fix: Adding input to workflow Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a3d85f598e9..21eb7442bbb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,17 @@ jobs: runs-on: ubuntu-latest outputs: highest_semver_tag: ${{ steps.strip_prefix.outputs.highest_semver_tag }} + steps: + - name: Strip 'v' Prefix for Highest SemVer Tag + id: strip_prefix + run: | + if [[ -z "${{ github.event.inputs.custom_version }}" ]]; then + echo "Error: custom_version input is required." + exit 1 + fi + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version#v }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Highest SemVer Tag: $HIGHEST_SEMVER_TAG" get-version: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest From e34bf65226f18d34d39bc27d6378ab95925b7206 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 21:48:03 -0500 Subject: [PATCH 10/90] chore: Updating workflow to use custom version for get highest semver step Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 21eb7442bbb..56c84a30554 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,21 +16,6 @@ on: type: string jobs: - prepare-versions: - runs-on: ubuntu-latest - outputs: - highest_semver_tag: ${{ steps.strip_prefix.outputs.highest_semver_tag }} - steps: - - name: Strip 'v' Prefix for Highest SemVer Tag - id: strip_prefix - run: | - if [[ -z "${{ github.event.inputs.custom_version }}" ]]; then - echo "Error: custom_version input is required." - exit 1 - fi - HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version#v }}" - echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" - echo "Highest SemVer Tag: $HIGHEST_SEMVER_TAG" get-version: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest @@ -93,10 +78,17 @@ jobs: env: RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} run: | - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" + else + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" + fi fi - name: Check output env: @@ -194,7 +186,7 @@ jobs: uses: ./.github/workflows/build_wheels.yml with: release_version: ${{ github.event.inputs.custom_version }} - highest_semver_tag: ${{ needs.prepare-versions.outputs.highest_semver_tag }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} publish-python-sdk: if: github.repository == 'feast-dev/feast' From 07b4c74fd9cca0a2dca47a06f97e1ce76290c3d4 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 22:10:38 -0500 Subject: [PATCH 11/90] chore: Fixing typo in publish workflow Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 56c84a30554..ec03669666d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,30 +31,16 @@ jobs: highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: - uses: actions/checkout@v4 - - name: Validate custom version input - id: validate_custom_version + - name: Get release version + id: get_release_version run: | if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + echo "Using custom version: ${{ github.event.inputs.custom_version }}" if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." exit 1 fi - echo "Validated custom version: ${{ github.event.inputs.custom_version }}" - fi - - name: Strip 'v' Prefix for Highest SemVer Tag - id: strip_prefix - run: | - HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version#v }}" - echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" - echo "Highest SemVer Tag: $HIGHEST_SEMVER_TAG" - - - - name: Get release version - id: get_release_version - run: | - if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then - echo "Using custom version: ${{ github.event.inputs.custom_version }}" echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then echo "Using tag reference: ${GITHUB_REF#refs/tags/}" From 3a2a9b0c83dc2d39d12e2301449118bb04f3fec0 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 22:24:20 -0500 Subject: [PATCH 12/90] chore: Consolidating more of Publish and adding checks to pass input args in build wheels workflows Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 14 ++++++++++---- .github/workflows/publish.yml | 5 +++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d23fd23325e..0d0b58f2bfc 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -51,10 +51,16 @@ jobs: env: RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} run: | - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + if [[ -n "${{ inputs.highest_semver_tag }}" ]]; then + echo "Using provided highest semver version: ${{ inputs.highest_semver_tag }}" + echo "::set-output name=highest_semver_tag::${{ inputs.highest_semver_tag }}" + else + echo "No release version provided. Falling back to infra/scripts/setup-common-functions.sh." + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + fi fi - name: Check output id: check_output diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ec03669666d..92b415ae125 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -170,9 +170,10 @@ jobs: build_wheels: uses: ./.github/workflows/build_wheels.yml + needs: get-version with: - release_version: ${{ github.event.inputs.custom_version }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + release_version: ${{ needs.get-version.outputs.release_version }} + highest_semver_tag: ${{ needs.get-version.outputs.highest_semver_tag }} publish-python-sdk: if: github.repository == 'feast-dev/feast' From e3fa4efd172dab1ade2dc025dfc8e2039271c30a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 22:48:27 -0500 Subject: [PATCH 13/90] chore: Adding exceptiong handling to build_wheels and additional logging Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 0d0b58f2bfc..41c78fd1c8d 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -206,11 +206,15 @@ jobs: # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of. - name: Validate Feast Version run: | + if ! VERSION_OUTPUT=$(feast version); then + echo "Error: Failed to get Feast version." + exit 1 + fi VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+' OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$' - VERSION_OUTPUT=$(feast version) VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX") OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX") + echo "Installed Feast Version: $VERSION and using Feast Version: $VERSION_WITHOUT_PREFIX" if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then echo "Correct Feast Version Installed" else From 4583d048d673a75c27212cdbcdffc6dcf216fd12 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 5 Dec 2024 23:05:29 -0500 Subject: [PATCH 14/90] chore: Adding feast version earlier in workflow Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 41c78fd1c8d..262a0b60b71 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -206,6 +206,7 @@ jobs: # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of. - name: Validate Feast Version run: | + feast version if ! VERSION_OUTPUT=$(feast version); then echo "Error: Failed to get Feast version." exit 1 From 3026c191595203c83a78dd9b2eaea1446f5cb5eb Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 12:02:30 -0500 Subject: [PATCH 15/90] chore: Renaming download artifact python-wheels Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 262a0b60b71..bda58e48e48 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -96,7 +96,7 @@ jobs: python -m build --wheel --outdir wheelhouse/ - uses: actions/upload-artifact@v4 with: - name: python-wheels + name: wheels path: ./wheelhouse/*.whl build-source-distribution: From 7999f2a9e75569aa3d66e657a6405671bef2d8f0 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 12:17:47 -0500 Subject: [PATCH 16/90] chore: Renaming upload artifact to python-wheels Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index bda58e48e48..262a0b60b71 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -96,7 +96,7 @@ jobs: python -m build --wheel --outdir wheelhouse/ - uses: actions/upload-artifact@v4 with: - name: wheels + name: python-wheels path: ./wheelhouse/*.whl build-source-distribution: From 7732b28ee4ec86c58f9169064d5aff0c1fd96983 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 14:32:22 -0500 Subject: [PATCH 17/90] chore: Updating build wheels to checkout latest tag Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 262a0b60b71..57e2a1698f7 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -118,6 +118,8 @@ jobs: - name: Build and install dependencies # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. run: | + git fetch --tags + git checkout ${{ needs.get-version.outputs.release_version }} pip install -U pip setuptools wheel twine make build-ui git status From 679e1bde7ffd9274147986ec644d33bf026249f5 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 14:48:03 -0500 Subject: [PATCH 18/90] chore: Hardcoding 0.42.0 to push to PyPI Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 57e2a1698f7..cdf95f8d312 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -119,7 +119,7 @@ jobs: # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. run: | git fetch --tags - git checkout ${{ needs.get-version.outputs.release_version }} + git checkout v0.42.0 pip install -U pip setuptools wheel twine make build-ui git status From f7ac71f0416bb925e35326980665332798aef507 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 14:55:41 -0500 Subject: [PATCH 19/90] chore: Update wheels as well to checkout proper version Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index cdf95f8d312..14ca70bcfc3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -92,6 +92,8 @@ jobs: run: make build-ui - name: Build wheels run: | + git fetch --tags + git checkout v0.42.0 python -m pip install build python -m build --wheel --outdir wheelhouse/ - uses: actions/upload-artifact@v4 From ad1375a270b0519c3d8f2e86844fb34ab4b2ff99 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 6 Dec 2024 15:04:20 -0500 Subject: [PATCH 20/90] chore: Rename artifact for downloading to publish python sdk Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 92b415ae125..fcd4a1b7201 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -182,7 +182,7 @@ jobs: steps: - uses: actions/download-artifact@v4.1.7 with: - name: wheels + name: python-wheels path: dist - uses: pypa/gh-action-pypi-publish@v1.4.2 with: From 82bc0c06612ccbc95d5bcf2f288dd267e74b8a9c Mon Sep 17 00:00:00 2001 From: Harri Lehtola <1781172+peruukki@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:44:09 +0200 Subject: [PATCH 21/90] chore: Upgrade react-router-dom and use-query-params to latest in /ui (#4764) --- ui/package.json | 4 +- ui/src/FeastUI.tsx | 9 ++-- ui/src/components/EuiCustomLink.jsx | 2 +- ui/src/components/ProjectSelector.test.tsx | 2 +- ui/src/hacks/RouteAdapter.ts | 39 ---------------- ui/src/test-utils.tsx | 12 +++-- ui/yarn.lock | 54 +++++++++++----------- 7 files changed, 41 insertions(+), 81 deletions(-) delete mode 100644 ui/src/hacks/RouteAdapter.ts diff --git a/ui/package.json b/ui/package.json index ea7e953edcf..ea911342091 100644 --- a/ui/package.json +++ b/ui/package.json @@ -35,9 +35,9 @@ "react-app-polyfill": "^3.0.0", "react-code-blocks": "^0.1.6", "react-query": "^3.39.3", - "react-router-dom": "<6.4.0", + "react-router-dom": "^6.28.0", "tslib": "^2.3.1", - "use-query-params": "^1.2.3", + "use-query-params": "^2.2.1", "zod": "^3.11.6" }, "scripts": { diff --git a/ui/src/FeastUI.tsx b/ui/src/FeastUI.tsx index 628b916f2d8..210ff7a4aa7 100644 --- a/ui/src/FeastUI.tsx +++ b/ui/src/FeastUI.tsx @@ -3,7 +3,7 @@ import React from "react"; import { BrowserRouter } from "react-router-dom"; import { QueryClient, QueryClientProvider } from "react-query"; import { QueryParamProvider } from "use-query-params"; -import RouteAdapter from "./hacks/RouteAdapter"; +import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import FeastUISansProviders, { FeastUIConfigs } from "./FeastUISansProviders"; interface FeastUIProps { @@ -17,11 +17,10 @@ const FeastUI = ({ reactQueryClient, feastUIConfigs }: FeastUIProps) => { const queryClient = reactQueryClient || defaultQueryClient; return ( - + // Disable v7_relativeSplatPath: custom tab routes don't currently work with it + - + diff --git a/ui/src/components/EuiCustomLink.jsx b/ui/src/components/EuiCustomLink.jsx index cf646d43f73..6872dae4275 100644 --- a/ui/src/components/EuiCustomLink.jsx +++ b/ui/src/components/EuiCustomLink.jsx @@ -39,7 +39,7 @@ export default function EuiCustomLink({ to, ...rest }) { } // Generate the correct link href (with basename accounted for) - const href = useHref({ pathname: to }); + const href = useHref(to); const props = { ...rest, href, onClick }; return ; diff --git a/ui/src/components/ProjectSelector.test.tsx b/ui/src/components/ProjectSelector.test.tsx index fc5b3c68400..dfaaab7f626 100644 --- a/ui/src/components/ProjectSelector.test.tsx +++ b/ui/src/components/ProjectSelector.test.tsx @@ -40,7 +40,7 @@ test("in a full App render, it shows the right initial project", async () => { name: "Top Level", }); - within(topLevelNavigation).getByDisplayValue("Credit Score Project"); + await within(topLevelNavigation).findByDisplayValue("Credit Score Project"); expect(options.length).toBe(1); diff --git a/ui/src/hacks/RouteAdapter.ts b/ui/src/hacks/RouteAdapter.ts deleted file mode 100644 index e7743c9d90b..00000000000 --- a/ui/src/hacks/RouteAdapter.ts +++ /dev/null @@ -1,39 +0,0 @@ -import React from "react"; -import { Location } from "history"; -import { - useLocation, - useNavigate, - Location as RouterLocation, -} from "react-router-dom"; - -// via: https://github.com/pbeshai/use-query-params/issues/196#issuecomment-996893750 -interface RouteAdapterProps { - children: React.FunctionComponent<{ - history: { - replace(location: Location): void; - push(location: Location): void; - }; - location: RouterLocation; - }>; -} - -// Via: https://github.com/pbeshai/use-query-params/blob/cd44e7fb3394620f757bfb09ff57b7f296d9a5e6/examples/react-router-6/src/index.js#L36 -const RouteAdapter = ({ children }: RouteAdapterProps) => { - const navigate = useNavigate(); - const location = useLocation(); - - const adaptedHistory = React.useMemo( - () => ({ - replace(location: Location) { - navigate(location, { replace: true, state: location.state }); - }, - push(location: Location) { - navigate(location, { replace: false, state: location.state }); - }, - }), - [navigate] - ); - return children && children({ history: adaptedHistory, location }); -}; - -export default RouteAdapter; diff --git a/ui/src/test-utils.tsx b/ui/src/test-utils.tsx index c180b01872c..0130686252d 100644 --- a/ui/src/test-utils.tsx +++ b/ui/src/test-utils.tsx @@ -2,8 +2,8 @@ import React from "react"; import { render, RenderOptions } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "react-query"; import { QueryParamProvider } from "use-query-params"; +import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import { MemoryRouter as Router } from "react-router-dom"; -import RouteAdapter from "./hacks/RouteAdapter"; interface ProvidersProps { children: React.ReactNode; @@ -14,10 +14,12 @@ const queryClient = new QueryClient(); const AllTheProviders = ({ children }: ProvidersProps) => { return ( - - + + {children} diff --git a/ui/yarn.lock b/ui/yarn.lock index 0057595f391..773260ad861 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -1869,7 +1869,7 @@ "@babel/plugin-transform-modules-commonjs" "^7.25.9" "@babel/plugin-transform-typescript" "^7.25.9" -"@babel/runtime@^7.0.0", "@babel/runtime@^7.12.13", "@babel/runtime@^7.7.6", "@babel/runtime@^7.9.2": +"@babel/runtime@^7.0.0", "@babel/runtime@^7.12.13", "@babel/runtime@^7.9.2": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.7.tgz#03ff99f64106588c9c403c6ecb8c3bafbbdff1fa" integrity sha512-9E9FJowqAsytyOY6LG+1KuueckRL+aQW+mKvXRXnuFGyRAyepJPmEo9vgMfXUA6O9u3IeEdv9MAkppFcaQwogQ== @@ -2709,6 +2709,11 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@remix-run/router@1.21.0": + version "1.21.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.21.0.tgz#c65ae4262bdcfe415dbd4f64ec87676e4a56e2b5" + integrity sha512-xfSkCAchbdG5PnbrKqFWwia4Bi61nH+wm8wLEqfHDyp7Y3dZzgqS2itV8i4gAq9pC2HsTpwyBC6Ds8VHZ96JlA== + "@rollup/plugin-babel@^5.2.0", "@rollup/plugin-babel@^5.3.1": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -6715,13 +6720,6 @@ highlightjs-vue@^1.0.0: resolved "https://registry.yarnpkg.com/highlightjs-vue/-/highlightjs-vue-1.0.0.tgz#fdfe97fbea6354e70ee44e3a955875e114db086d" integrity sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA== -history@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/history/-/history-5.2.0.tgz#7cdd31cf9bac3c5d31f09c231c9928fad0007b7c" - integrity sha512-uPSF6lAJb3nSePJ43hN3eKj1dTWpN9gMod0ZssbFTIsen+WehTmEadgL+kg78xLJFdRfrrC//SavDzmRVdE+Ig== - dependencies: - "@babel/runtime" "^7.7.6" - hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" @@ -9962,20 +9960,20 @@ react-remove-scroll@^2.6.0: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@<6.4.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" - integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== +react-router-dom@^6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.28.0.tgz#f73ebb3490e59ac9f299377062ad1d10a9f579e6" + integrity sha512-kQ7Unsl5YdyOltsPGl31zOjLrDv+m2VcIEcIHqYYD3Lp0UppLjrzcfJqDJwXxFw3TH/yvapbnUvPlAj7Kx5nbg== dependencies: - history "^5.2.0" - react-router "6.3.0" + "@remix-run/router" "1.21.0" + react-router "6.28.0" -react-router@6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557" - integrity sha512-7Wh1DzVQ+tlFjkeo+ujvjSqSJmkt1+8JO+T5xklPlgrh70y7ogx75ODRW0ThWhY7S+6yEDks8TYrtQe/aoboBQ== +react-router@6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.28.0.tgz#29247c86d7ba901d7e5a13aa79a96723c3e59d0d" + integrity sha512-HrYdIFqdrnhDw0PqG/AKjAqEqM7AvxCz0DQ4h2W8k6nqmc5uRBYDag0SBxx9iYz5G8gnuNVLzUe13wl9eAsXXg== dependencies: - history "^5.2.0" + "@remix-run/router" "1.21.0" react-style-singleton@^2.2.1: version "2.2.1" @@ -10588,10 +10586,10 @@ serialize-javascript@^6.0.0, serialize-javascript@^6.0.1: dependencies: randombytes "^2.1.0" -serialize-query-params@^1.3.5: - version "1.3.6" - resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-1.3.6.tgz#5dd5225db85ce747fe6fbc4897628504faafec6d" - integrity sha512-VlH7sfWNyPVZClPkRacopn6sn5uQMXBsjPVz1+pBHX895VpcYVznfJtZ49e6jymcrz+l/vowkepCZn/7xEAEdw== +serialize-query-params@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-2.0.2.tgz#598a3fb9e13f4ea1c1992fbd20231aa16b31db81" + integrity sha512-1chMo1dST4pFA9RDXAtF0Rbjaut4is7bzFbI1Z26IuMub68pNCILku85aYmeFhvnY//BXUPUhoRMjYcsT93J/Q== serve-index@^1.9.1: version "1.9.1" @@ -11709,12 +11707,12 @@ use-memo-one@^1.1.3: resolved "https://registry.yarnpkg.com/use-memo-one/-/use-memo-one-1.1.3.tgz#2fd2e43a2169eabc7496960ace8c79efef975e99" integrity sha512-g66/K7ZQGYrI6dy8GLpVcMsBp4s17xNkYJVSMvTEevGy3nDxHOfE6z8BVE22+5G5x7t3+bhzrlTDB7ObrEE0cQ== -use-query-params@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-1.2.3.tgz#306c31a0cbc714e8a3b4bd7e91a6a9aaccaa5e22" - integrity sha512-cdG0tgbzK+FzsV6DAt2CN8Saa3WpRnze7uC4Rdh7l15epSFq7egmcB/zuREvPNwO5Yk80nUpDZpiyHsoq50d8w== +use-query-params@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-2.2.1.tgz#c558ab70706f319112fbccabf6867b9f904e947d" + integrity sha512-i6alcyLB8w9i3ZK3caNftdb+UnbfBRNPDnc89CNQWkGRmDrm/gfydHvMBfVsQJRq3NoHOM2dt/ceBWG2397v1Q== dependencies: - serialize-query-params "^1.3.5" + serialize-query-params "^2.0.2" use-sidecar@^1.1.2: version "1.1.2" From 41de0c317f7ba7036f9bd2c214ae592b9fb392d4 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 7 Dec 2024 10:01:39 -0500 Subject: [PATCH 22/90] chore: Adding manual publish images file Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish_images.yml | 138 +++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 .github/workflows/publish_images.yml diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml new file mode 100644 index 00000000000..4005202ec42 --- /dev/null +++ b/.github/workflows/publish_images.yml @@ -0,0 +1,138 @@ +name: build and publish docker images + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GIT_AUTHOR_NAME: feast-ci-bot + GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co + GIT_COMMITTER_NAME: feast-ci-bot + GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co + outputs: + release_version: ${{ steps.get_release_version.outputs.release_version }} + version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + steps: + - uses: actions/checkout@v4 + - name: Get release version + id: get_release_version + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + echo "Using custom version: ${{ github.event.inputs.custom_version }}" + if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then + echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." + exit 1 + fi + echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" + elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then + echo "Using tag reference: ${GITHUB_REF#refs/tags/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" + else + echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" + fi + - name: Get release version without prefix + id: get_release_version_without_prefix + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ "${RELEASE_VERSION}" == v* ]]; then + echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" + else + echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" + fi + - name: Get highest semver + id: get_highest_semver + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" + else + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" + fi + fi + - name: Check output + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + run: | + echo $RELEASE_VERSION + echo $VERSION_WITHOUT_PREFIX + echo $HIGHEST_SEMVER_TAG + + build-publish-docker-images: + runs-on: ubuntu-latest + needs: [get-version, publish-python-sdk] + strategy: + matrix: + component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator] + env: + MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar + REGISTRY: feastdev + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Use gcloud CLI + run: gcloud info + - run: gcloud auth configure-docker --quiet + - name: Build image + run: | + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + - name: Push versioned images + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + run: | + make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest + docker push feastdev/${{ matrix.component }}:latest + fi + From 28d91b60b75f926060fceec2e2f73fc2e51f46aa Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 7 Dec 2024 10:03:26 -0500 Subject: [PATCH 23/90] chore: Fixing manual publish image workflow Signed-off-by: Francisco Javier Arceo --- .github/workflows/publish_images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 4005202ec42..a80036cc9ff 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -85,7 +85,7 @@ jobs: build-publish-docker-images: runs-on: ubuntu-latest - needs: [get-version, publish-python-sdk] + needs: [get-version] strategy: matrix: component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator] From 3d6bf42b83492a1d2f1e4e642ab1c8e09efccb6a Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:21:45 -0500 Subject: [PATCH 24/90] docs: Added missing documentation for the registry server and remote client (#4825) * Added missing documentation for the registry server Signed-off-by: Theodor Mihalache * Changes following review Signed-off-by: Theodor Mihalache * Changes following review Signed-off-by: Theodor Mihalache --------- Signed-off-by: Theodor Mihalache --- docs/reference/feature-servers/README.md | 6 +++- .../feature-servers/registry-server.md | 26 +++++++++++++++++ docs/reference/registries/remote.md | 28 +++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docs/reference/feature-servers/registry-server.md create mode 100644 docs/reference/registries/remote.md diff --git a/docs/reference/feature-servers/README.md b/docs/reference/feature-servers/README.md index 2ceaf5807f3..156e60c7431 100644 --- a/docs/reference/feature-servers/README.md +++ b/docs/reference/feature-servers/README.md @@ -1,4 +1,4 @@ -# Feature servers +# Feast servers Feast users can choose to retrieve features from a feature server, as opposed to through the Python SDK. @@ -12,4 +12,8 @@ Feast users can choose to retrieve features from a feature server, as opposed to {% content-ref url="offline-feature-server.md" %} [offline-feature-server.md](offline-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="registry-server.md" %} +[registry-server.md](registry-server.md) {% endcontent-ref %} \ No newline at end of file diff --git a/docs/reference/feature-servers/registry-server.md b/docs/reference/feature-servers/registry-server.md new file mode 100644 index 00000000000..9707a597035 --- /dev/null +++ b/docs/reference/feature-servers/registry-server.md @@ -0,0 +1,26 @@ +# Registry server + +## Description + +The Registry server uses the gRPC communication protocol to exchange data. +This enables users to communicate with the server using any programming language that can make gRPC requests. + +## How to configure the server + +## CLI + +There is a CLI command that starts the Registry server: `feast serve_registry`. By default, remote Registry Server uses port 6570, the port can be overridden with a `--port` flag. +To start the Registry Server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments. +More info about TLS mode can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#starting-feast-registry-server-in-tls-mode) + +## How to configure the client + +Please see the detail how to configure Remote Registry client [remote.md](../registries/remote.md) + +# Registry Server Permissions and Access Control + +Please refer the [page](./../registry/registry-permissions.md) for more details on API Endpoints and Permissions. + +## How to configure Authentication and Authorization ? + +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. \ No newline at end of file diff --git a/docs/reference/registries/remote.md b/docs/reference/registries/remote.md new file mode 100644 index 00000000000..3651aeb71ea --- /dev/null +++ b/docs/reference/registries/remote.md @@ -0,0 +1,28 @@ +# Remote Registry + +## Description + +The Remote Registry is a gRPC client for the registry that implements the `RemoteRegistry` class using the existing `BaseRegistry` interface. + +## How to configure the client + +User needs to create a client side `feature_store.yaml` file, set the `registry_type` to `remote` and provide the server connection configuration. +The `path` parameter is a URL with a port (default is 6570) used by the client to connect with the Remote Registry server. + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_type: remote + path: http://localhost:6570 +``` +{% endcode %} + +The optional `cert` parameter can be configured as well, it should point to the public certificate path when the Registry Server starts in SSL mode. This may be needed if the Registry Server is started with a self-signed certificate, typically this file ends with *.crt, *.cer, or *.pem. +More info about the `cert` parameter can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#feast-client-connecting-to-remote-registry-sever-started-in-tls-mode) + +## How to configure the server + +Please see the detail how to configure registry server [registry-server.md](../feature-servers/registry-server.md) + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. From d558ef7e19aa561c37c38d4d0da2b8c1467414f5 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:27:12 -0700 Subject: [PATCH 25/90] feat: Operator E2E test to validate FeatureStore custom resource using remote registry (#4822) * Added new e2e test case to do the remote registry deployment. abstracted the code to do the featurestore testing. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * abstracted the code even further. Now the custom resource CR will execute only once for all the test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fixing the operator e2e test. this was commented before by mistake. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * increasing the remote registry deployment timeout to see if it solves the github CI timeout issue. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Trying to increase the timeout and also adding the debugging actions when there is a failure. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * incorporating the code review comments. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * incorporating the code review comments by using the feastRef. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * incorporating the code review comments by marshaling json to the go struct object rather than map. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the lint error. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../operator-e2e-integration-tests.yml | 11 +- infra/feast-operator/Makefile | 2 +- infra/feast-operator/test/e2e/e2e_test.go | 271 ++++++++++-------- infra/feast-operator/test/e2e/test_util.go | 46 +++ ...v1alpha1_remote_registry_featurestore.yaml | 16 ++ 5 files changed, 230 insertions(+), 116 deletions(-) create mode 100644 infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index 23c250cc535..cbb505c3fe8 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -13,6 +13,7 @@ on: jobs: operator-e2e-tests: + timeout-minutes: 40 if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && @@ -38,7 +39,7 @@ jobs: - name: Create KIND cluster run: | - kind create cluster --name $KIND_CLUSTER --wait 5m + kind create cluster --name $KIND_CLUSTER --wait 10m - name: Set up kubernetes context run: | @@ -51,8 +52,16 @@ jobs: cd infra/feast-operator/ make test-e2e + - name: Debug KIND Cluster when there is a failure + if: failure() + run: | + kubectl get pods --all-namespaces + kubectl describe nodes + - name: Clean up if: always() run: | # Delete the KIND cluster after tests kind delete cluster --name kind-$KIND_CLUSTER + + diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 310d64afaaa..6984ac66e7f 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -117,7 +117,7 @@ test: build-installer fmt vet lint envtest ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: - go test ./test/e2e/ -v -ginkgo.v + go test -timeout 30m ./test/e2e/ -v -ginkgo.v .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index 7d9fb9af056..23637ff224b 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -28,145 +28,188 @@ import ( ) const feastControllerNamespace = "feast-operator-system" +const timeout = 2 * time.Minute +const controllerDeploymentName = "feast-operator-controller-manager" var _ = Describe("controller", Ordered, func() { BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace) _, _ = utils.Run(cmd) + var err error + // projectimage stores the name of the image used in the example + var projectimage = "localhost/feast-operator:v0.0.1" + + By("building the manager(Operator) image") + cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectimage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("building the feast image") + cmd = exec.Command("make", "feast-ci-dev-docker-img") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + // this image will be built in above make target. + var feastImage = "feastdev/feature-server:dev" + var feastLocalImage = "localhost/feastdev/feature-server:dev" + + By("Tag the local feast image for the integration tests") + cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the feast image on Kind cluster") + err = utils.LoadImageToKindClusterWithName(feastLocalImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Validating that the controller-manager deployment is in available state") + err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) + Expect(err).To(BeNil(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + controllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) }) AfterAll(func() { //Add any post clean up code here. + By("Uninstalling the feast CRD") + cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - Context("Operator", func() { + Context("Operator E2E Tests", func() { It("Should be able to deploy and run a default feature store CR successfully", func() { - //var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "localhost/feast-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("building the feast image") - cmd = exec.Command("make", "feast-ci-dev-docker-img") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // this image will be built in above make target. - var feastImage = "feastdev/feature-server:dev" - var feastLocalImage = "localhost/feastdev/feature-server:dev" - - By("Tag the local feast image for the integration tests") - cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the feast image on Kind cluster") - err = utils.LoadImageToKindClusterWithName(feastLocalImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - timeout := 2 * time.Minute - - controllerDeploymentName := "feast-operator-controller-manager" - By("Validating that the controller-manager deployment is in available state") - err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - controllerDeploymentName, err, - )) - fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) - By("deploying the Simple Feast Custom Resource to Kubernetes") - cmd = exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) _, cmdOutputerr := utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - namespace := "default" - - deploymentNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, deploymentName := range deploymentNames { - By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) - err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - deploymentName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", deploymentName) - } - - By("Check if the feast client - kubernetes config map exists.") - configMapName := "feast-simple-feast-setup-client" - err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).To(BeNil(), fmt.Sprintf( - "config map %s is not available but expected to be available. \nError: %v\n", - configMapName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", configMapName) - - serviceAccountNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceAccountName := range serviceAccountNames { - By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) - err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).To(BeNil(), fmt.Sprintf( - "Service account %s does not exist in namespace %s. Error: %v", - serviceAccountName, namespace, err, - )) - fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) - } - - serviceNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceName := range serviceNames { - By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) - err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).To(BeNil(), fmt.Sprintf( - "kubernetes service %s is not available but expected to be available. \nError: %v\n", - serviceName, err, - )) - fmt.Printf("kubernetes service %s is available\n", serviceName) - } - - By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", "simple-feast-setup")) - err = checkIfFeatureStoreCustomResourceConditionsInReady("simple-feast-setup", namespace) - Expect(err).To(BeNil(), fmt.Sprintf( - "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", - "simple-feast-setup", err, - )) - fmt.Printf("FeatureStore customer resource %s conditions are in Ready State\n", "simple-feast-setup") + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) By("deleting the feast deployment") cmd = exec.Command("kubectl", "delete", "-f", "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") _, cmdOutputerr = utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + }) + + It("Should be able to deploy and run a feature store with remote registry CR successfully", func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr := utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) + + var remoteRegistryNs = "remote-registry" + cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) + _, _ = utils.Run(cmd) + + By("deploying the Simple Feast remote registry Custom Resource to Kubernetes") + cmd = exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - By("Uninstalling the feast CRD") - cmd = exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + remoteFeatureStoreName := "simple-feast-remote-setup" + + validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout) + + By("deleting the feast remote registry deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) }) }) }) + +func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { + hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) + Expect(err).To(BeNil(), fmt.Sprintf( + "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", + featureStoreName, err)) + + k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName), + fmt.Sprintf("feast-%s-offline", featureStoreName), + } + + if !hasRemoteRegistry { + k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName)) + } + + for _, deploymentName := range k8ResourceNames { + By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) + err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) + Expect(err).To(BeNil(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + deploymentName, err, + )) + fmt.Printf("Feast Deployment %s is available\n", deploymentName) + } + + By("Check if the feast client - kubernetes config map exists.") + configMapName := fmt.Sprintf("feast-%s-client", featureStoreName) + err = checkIfConfigMapExists(namespace, configMapName) + Expect(err).To(BeNil(), fmt.Sprintf( + "config map %s is not available but expected to be available. \nError: %v\n", + configMapName, err, + )) + fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) + + for _, serviceAccountName := range k8ResourceNames { + By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) + err = checkIfServiceAccountExists(namespace, serviceAccountName) + Expect(err).To(BeNil(), fmt.Sprintf( + "Service account %s does not exist in namespace %s. Error: %v", + serviceAccountName, namespace, err, + )) + fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) + } + + for _, serviceName := range k8ResourceNames { + By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) + err = checkIfKubernetesServiceExists(namespace, serviceName) + Expect(err).To(BeNil(), fmt.Sprintf( + "kubernetes service %s is not available but expected to be available. \nError: %v\n", + serviceName, err, + )) + fmt.Printf("kubernetes service %s is available\n", serviceName) + } + + By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) + err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) + Expect(err).To(BeNil(), fmt.Sprintf( + "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", + featureStoreName, err, + )) + fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) +} diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index f30d8cbebf5..7d44ac1296a 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -3,10 +3,13 @@ package e2e import ( "bytes" "encoding/json" + "errors" "fmt" "os/exec" "strings" "time" + + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" ) // dynamically checks if all conditions of custom resource featurestore are in "Ready" state. @@ -184,3 +187,46 @@ func checkIfKubernetesServiceExists(namespace, serviceName string) error { return nil } + +func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, + "-o=jsonpath='{.spec.services.registry}'") + + // Capture the output + output, err := cmd.Output() + if err != nil { + return false, err // Return false on command execution failure + } + + // Convert output to string and trim any extra spaces + result := strings.TrimSpace(string(output)) + + // Remove single quotes if present + if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { + result = strings.Trim(result, "'") + } + + if result == "" { + return false, errors.New("kubectl get featurestore command returned empty output") + } + + // Parse the JSON into a map + var registryConfig v1alpha1.Registry + if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { + return false, err // Return false on JSON parsing failure + } + + if registryConfig.Remote == nil { + return false, nil + } + + hasHostname := registryConfig.Remote.Hostname != nil + hasValidFeastRef := registryConfig.Remote.FeastRef != nil && + registryConfig.Remote.FeastRef.Name != "" + + if hasHostname || hasValidFeastRef { + return true, nil + } + + return false, nil +} diff --git a/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml new file mode 100644 index 00000000000..61c010f0576 --- /dev/null +++ b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml @@ -0,0 +1,16 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: simple-feast-remote-setup +spec: + feastProject: my_project + services: + onlineStore: + image: 'localhost/feastdev/feature-server:dev' + offlineStore: + image: 'localhost/feastdev/feature-server:dev' + registry: + remote: + feastRef: + name: simple-feast-setup + namespace: default \ No newline at end of file From 732865f20e7fae7a46f54be7bc469ce2b3bc44e2 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:24:33 -0700 Subject: [PATCH 26/90] feat: Go Operator - Parsing the output to go structs (#4832) Now parsing the output to go lang structs rather than a Map to simplify the parsing logic. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- infra/feast-operator/test/e2e/e2e_test.go | 3 +- infra/feast-operator/test/e2e/test_util.go | 54 +++++----------------- 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index 23637ff224b..fdf58d8f3b7 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -124,10 +124,11 @@ var _ = Describe("controller", Ordered, func() { validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) var remoteRegistryNs = "remote-registry" + By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) _, _ = utils.Run(cmd) - By("deploying the Simple Feast remote registry Custom Resource to Kubernetes") + By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") cmd = exec.Command("kubectl", "apply", "-f", "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) _, cmdOutputerr = utils.Run(cmd) diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index 7d44ac1296a..d92f719fb97 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -9,6 +9,8 @@ import ( "strings" "time" + appsv1 "k8s.io/api/apps/v1" + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" ) @@ -26,36 +28,17 @@ func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namesp featureStoreName, namespace, err, stderr.String()) } - // Parse the JSON into a generic map - var resource map[string]interface{} + // Parse the JSON into FeatureStore + var resource v1alpha1.FeatureStore if err := json.Unmarshal(out.Bytes(), &resource); err != nil { return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) } - // Traverse the JSON structure to extract conditions - status, ok := resource["status"].(map[string]interface{}) - if !ok { - return fmt.Errorf("status field is missing or invalid in the resource JSON") - } - - conditions, ok := status["conditions"].([]interface{}) - if !ok { - return fmt.Errorf("conditions field is missing or invalid in the status section") - } - // Validate all conditions - for _, condition := range conditions { - conditionMap, ok := condition.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid condition format") - } - - conditionType := conditionMap["type"].(string) - conditionStatus := conditionMap["status"].(string) - - if conditionStatus != "True" { + for _, condition := range resource.Status.Conditions { + if condition.Status != "True" { return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", - featureStoreName, conditionType, conditionStatus) + featureStoreName, condition.Type, condition.Status) } } @@ -87,30 +70,15 @@ func checkIfDeploymentExistsAndAvailable(namespace string, deploymentName string continue } - // Parse the JSON output into a map - var result map[string]interface{} + // Parse the JSON output into Deployment + var result appsv1.Deployment if err := json.Unmarshal(output.Bytes(), &result); err != nil { return fmt.Errorf("failed to parse deployment JSON: %v", err) } - // Navigate to status.conditions - status, ok := result["status"].(map[string]interface{}) - if !ok { - return fmt.Errorf("failed to get status field from deployment JSON") - } - - conditions, ok := status["conditions"].([]interface{}) - if !ok { - return fmt.Errorf("failed to get conditions field from deployment JSON") - } - // Check for Available condition - for _, condition := range conditions { - cond, ok := condition.(map[string]interface{}) - if !ok { - continue - } - if cond["type"] == "Available" && cond["status"] == "True" { + for _, condition := range result.Status.Conditions { + if condition.Type == "Available" && condition.Status == "True" { return nil // Deployment is available } } From cdc075360242bfdf3812d394a3c9c550f81b0f98 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:48:00 -0500 Subject: [PATCH 27/90] fix: Made fixes to Go Operator DB persistence (#4830) * Made fixes to Go Operator DB persistence Signed-off-by: Theodor Mihalache * Fixes following review Signed-off-by: Theodor Mihalache --------- Signed-off-by: Theodor Mihalache --- .../api/v1alpha1/featurestore_types.go | 15 ++- .../crd/bases/feast.dev_featurestores.yaml | 18 ++- infra/feast-operator/dist/install.yaml | 18 ++- .../featurestore_controller_db_store_test.go | 117 +++++++++--------- .../controller/services/repo_config.go | 42 ++++--- .../controller/services/repo_config_test.go | 8 +- .../test/api/featurestore_types_test.go | 4 +- 7 files changed, 128 insertions(+), 94 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 0b516630cd2..454c909f7d0 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -99,7 +99,7 @@ type OfflineStorePersistence struct { // OfflineStoreFilePersistence configures the file-based persistence for the offline store service type OfflineStoreFilePersistence struct { - // +kubebuilder:validation:Enum=dask;duckdb + // +kubebuilder:validation:Enum=file;dask;duckdb Type string `json:"type,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` } @@ -107,11 +107,12 @@ type OfflineStoreFilePersistence struct { var ValidOfflineStoreFilePersistenceTypes = []string{ "dask", "duckdb", + "file", } // OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service type OfflineStoreDBStorePersistence struct { - // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;feast_trino.trino.TrinoOfflineStore;redis + // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;trino;redis;athena;mssql Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -125,8 +126,10 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ "redshift", "spark", "postgres", - "feast_trino.trino.TrinoOfflineStore", + "trino", "redis", + "athena", + "mssql", } // OnlineStore configures the deployed online store service @@ -158,7 +161,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service type OnlineStoreDBStorePersistence struct { - // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore + // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -178,6 +181,10 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "mysql", "hazelcast", "singlestore", + "hbase", + "elasticsearch", + "qdrant", + "couchbase", } // LocalRegistryConfig configures the deployed registry service diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 6929796d34f..61b3e7adcf0 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -323,6 +323,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -355,8 +356,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -729,6 +732,10 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase type: string required: - secretRef @@ -1559,6 +1566,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -1592,8 +1600,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -1973,6 +1983,10 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase type: string required: - secretRef diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 9e213994eba..5d56e786394 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -331,6 +331,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -363,8 +364,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -737,6 +740,10 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase type: string required: - secretRef @@ -1567,6 +1574,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -1600,8 +1608,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -1981,6 +1991,10 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase type: string required: - secretRef diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 60235fe687e..377ee6bc512 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -78,7 +78,7 @@ sqlalchemy_config_kwargs: pool_pre_ping: true ` -var invalidSecretContainingTypeYamlString = ` +var secretContainingValidTypeYamlString = ` type: cassandra hosts: - 192.168.1.1 @@ -305,37 +305,12 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err.Error()).To(Equal("secret key invalid.secret.key doesn't exist in secret online-store-secret")) - By("Referring to a secret that contains parameter named type") - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - secret := &corev1.Secret{} - err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretContainingTypeYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"} - resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = "" - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).To(HaveOccurred()) - - Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type")) - By("Referring to a secret that contains parameter named type with invalid value") resource = &feastdevv1alpha1.FeatureStore{} err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) - secret = &corev1.Secret{} + secret := &corev1.Secret{} err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) Expect(err).NotTo(HaveOccurred()) secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretTypeYamlString) @@ -353,39 +328,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type")) - - By("Referring to a secret that contains parameter named registry_type") - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - secret = &corev1.Secret{} - err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - secret = &corev1.Secret{} - err = k8sClient.Get(ctx, registrySecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data["sql_custom_registry_key"] = nil - secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"} - resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = "" - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).To(HaveOccurred()) - - Expect(err.Error()).To(Equal("secret key sql in secret registry-store-secret contains invalid tag named registry_type")) + Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains tag named type with value wrong")) }) It("should successfully reconcile the resource", func() { @@ -506,6 +449,60 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort)))) + + By("Referring to a secret that contains parameter named type") + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + secret := &corev1.Secret{} + err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(secretContainingValidTypeYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"} + resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = "" + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + + Expect(err).To(Not(HaveOccurred())) + + By("Referring to a secret that contains parameter named registry_type") + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + secret = &corev1.Secret{} + err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + secret = &corev1.Secret{} + err = k8sClient.Get(ctx, registrySecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data["sql_custom_registry_key"] = nil + secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"} + resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = "" + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(Not(HaveOccurred())) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 22052aa724d..c70996ab867 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -50,7 +50,7 @@ func (feast *FeastServices) getServiceRepoConfig(feastType FeastServiceType) (Re func getServiceRepoConfig( feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, - secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { appliedSpec := featureStore.Status.Applied repoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) @@ -59,9 +59,9 @@ func getServiceRepoConfig( } if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { - propertiesMap, err := secretExtractionFunc(appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") - if err != nil { - return repoConfig, err + propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") + if authSecretErr != nil { + return repoConfig, authSecretErr } oidcServerProperties := map[string]interface{}{} @@ -109,7 +109,7 @@ func getServiceRepoConfig( return repoConfig, nil } -func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { +func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { repoConfig.Registry = RegistryConfig{} repoConfig.Registry.Path = DefaultRegistryEphemeralPath registryPersistence := services.Registry.Local.Persistence @@ -129,7 +129,7 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr if len(secretKeyName) == 0 { secretKeyName = string(repoConfig.Registry.RegistryType) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -149,7 +149,7 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr return nil } -func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { +func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { repoConfig.OnlineStore = OnlineStoreConfig{} repoConfig.OnlineStore.Path = DefaultOnlineStoreEphemeralPath @@ -170,7 +170,7 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret secretKeyName = string(repoConfig.OnlineStore.Type) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -187,7 +187,7 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret return nil } -func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { +func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { repoConfig.OfflineStore = OfflineStoreConfig{} repoConfig.OfflineStore.Type = OfflineFilePersistenceDaskConfigType offlineStorePersistence := services.OfflineStore.Persistence @@ -205,7 +205,7 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre secretKeyName = string(repoConfig.OfflineStore.Type) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -224,7 +224,7 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre return nil } -func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) ([]byte, error) { +func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) ([]byte, error) { clientRepo, err := getClientRepoConfig(feast.Handler.FeatureStore, secretExtractionFunc) if err != nil { return []byte{}, err @@ -234,7 +234,7 @@ func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func( func getClientRepoConfig( featureStore *feastdevv1alpha1.FeatureStore, - secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { status := featureStore.Status appliedServices := status.Applied.Services clientRepoConfig := RepoConfig{ @@ -292,7 +292,7 @@ func getClientRepoConfig( Type: OidcAuthType, } - propertiesMap, err := secretExtractionFunc(status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") + propertiesMap, err := secretExtractionFunc("", status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") if err != nil { return clientRepoConfig, err } @@ -318,7 +318,7 @@ func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) strin return path.Join(pvcConfig.MountPath, filePath) } -func (feast *FeastServices) extractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func (feast *FeastServices) extractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { secret, err := feast.getSecret(secretRef) if err != nil { return nil, err @@ -330,18 +330,20 @@ func (feast *FeastServices) extractConfigFromSecret(secretRef string, secretKeyN if !exists { return nil, fmt.Errorf("secret key %s doesn't exist in secret %s", secretKeyName, secretRef) } + err = yaml.Unmarshal(val, ¶meters) if err != nil { return nil, fmt.Errorf("secret %s contains invalid value", secretKeyName) } - _, exists = parameters["type"] - if exists { - return nil, fmt.Errorf("secret key %s in secret %s contains invalid tag named type", secretKeyName, secretRef) + + typeVal, typeExists := parameters["type"] + if typeExists && storeType != typeVal { + return nil, fmt.Errorf("secret key %s in secret %s contains tag named type with value %s", secretKeyName, secretRef, typeVal) } - _, exists = parameters["registry_type"] - if exists { - return nil, fmt.Errorf("secret key %s in secret %s contains invalid tag named registry_type", secretKeyName, secretRef) + typeVal, typeExists = parameters["registry_type"] + if typeExists && storeType != typeVal { + return nil, fmt.Errorf("secret key %s in secret %s contains tag named registry_type with value %s", secretKeyName, secretRef, typeVal) } } else { for k, v := range secret.Data { diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index b148f904706..7f017f4d102 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -486,17 +486,17 @@ func minimalFeatureStoreWithAllServices() *feastdevv1alpha1.FeatureStore { return feast } -func emptyMockExtractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func emptyMockExtractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return map[string]interface{}{}, nil } -func mockExtractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func mockExtractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return createParameterMap(), nil } func mockOidcConfigFromSecret( - oidcProperties map[string]interface{}) func(secretRef string, secretKeyName string) (map[string]interface{}, error) { - return func(secretRef string, secretKeyName string) (map[string]interface{}, error) { + oidcProperties map[string]interface{}) func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { + return func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return oidcProperties, nil } } diff --git a/infra/feast-operator/test/api/featurestore_types_test.go b/infra/feast-operator/test/api/featurestore_types_test.go index 302abef9384..126991266d3 100644 --- a/infra/feast-operator/test/api/featurestore_types_test.go +++ b/infra/feast-operator/test/api/featurestore_types_test.go @@ -377,7 +377,7 @@ var _ = Describe("FeatureStore API", func() { }) It("should fail when db persistence type is invalid", func() { - attemptInvalidCreationAndAsserts(ctx, onlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.online\", \"redis\", \"ikv\", \"datastore\", \"dynamodb\", \"bigtable\", \"postgres\", \"cassandra\", \"mysql\", \"hazelcast\", \"singlestore\"") + attemptInvalidCreationAndAsserts(ctx, onlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.online\", \"redis\", \"ikv\", \"datastore\", \"dynamodb\", \"bigtable\", \"postgres\", \"cassandra\", \"mysql\", \"hazelcast\", \"singlestore\", \"hbase\", \"elasticsearch\", \"qdrant\", \"couchbase\"") }) }) @@ -388,7 +388,7 @@ var _ = Describe("FeatureStore API", func() { attemptInvalidCreationAndAsserts(ctx, offlineStoreWithUnmanagedFileType(featurestore), "Unsupported value") }) It("should fail when db persistence type is invalid", func() { - attemptInvalidCreationAndAsserts(ctx, offlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.offline\", \"bigquery\", \"redshift\", \"spark\", \"postgres\", \"feast_trino.trino.TrinoOfflineStore\", \"redis\"") + attemptInvalidCreationAndAsserts(ctx, offlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.offline\", \"bigquery\", \"redshift\", \"spark\", \"postgres\", \"trino\", \"redis\", \"athena\", \"mssql\"") }) }) From f565565e0132ea5170221dc6af2e93a5dc3e750d Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 11 Dec 2024 13:32:26 -0600 Subject: [PATCH 28/90] fix: Add k8s module to feature-server image (#4839) add k8s module to feature-server image Signed-off-by: Tommy Hughes --- sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index f6bcbae8cd0..c1da48f55d0 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11-slim-bullseye RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio]" +RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s]" RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ From b8ede2ac6525b498d10e1231f955624d5e7c817b Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 11 Dec 2024 14:54:01 -0500 Subject: [PATCH 29/90] chore: Issue warning announcing entity's value_type as mandatory (#4833) * feat: Make entity value_type mandatory with deprecation warning - Add deprecation warning when value_type is not specified for an entity - Add test cases to verify deprecation warning behavior - Prepare for making value_type mandatory in next release Issue: feast-dev/feast#4670 Co-Authored-By: Francisco Javier Arceo Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: Fix import sorting in entity files - Reorder imports according to PEP8 - Group standard library imports together - Fix ruff linting issues Co-Authored-By: Francisco Javier Arceo Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sdk/python/feast/entity.py | 8 ++++++++ sdk/python/tests/unit/test_entity.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 290e6307a42..9c529115c8e 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings from datetime import datetime from typing import Dict, List, Optional @@ -79,6 +80,13 @@ def __init__( ValueError: Parameters are specified incorrectly. """ self.name = name + if value_type is None: + warnings.warn( + "Entity value_type will be mandatory in the next release. " + "Please specify a value_type for entity '%s'." % name, + DeprecationWarning, + stacklevel=2, + ) self.value_type = value_type or ValueType.UNKNOWN if join_keys and len(join_keys) > 1: diff --git a/sdk/python/tests/unit/test_entity.py b/sdk/python/tests/unit/test_entity.py index 78f71231049..b36f363a6ff 100644 --- a/sdk/python/tests/unit/test_entity.py +++ b/sdk/python/tests/unit/test_entity.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings + import assertpy import pytest @@ -73,3 +75,16 @@ def test_hash(): s4 = {entity1, entity2, entity3, entity4} assert len(s4) == 3 + + +def test_entity_without_value_type_warns(): + with pytest.warns(DeprecationWarning, match="Entity value_type will be mandatory"): + entity = Entity(name="my-entity") + assert entity.value_type == ValueType.UNKNOWN + + +def test_entity_with_value_type_no_warning(): + with warnings.catch_warnings(): + warnings.simplefilter("error") + entity = Entity(name="my-entity", value_type=ValueType.STRING) + assert entity.value_type == ValueType.STRING From d720cdf4bdb5f78723591fd20d44102a36c7a4da Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 11 Dec 2024 15:03:01 -0500 Subject: [PATCH 30/90] chore: Skip tests on doc updates (#4834) * feat: Make entity value_type mandatory with deprecation warning - Add deprecation warning when value_type is not specified for an entity - Add test cases to verify deprecation warning behavior - Prepare for making value_type mandatory in next release Issue: feast-dev/feast#4670 Co-Authored-By: Francisco Javier Arceo Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: Fix import sorting in entity files - Reorder imports according to PEP8 - Group standard library imports together - Fix ruff linting issues Co-Authored-By: Francisco Javier Arceo Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: Skip tests for community/docs/examples paths Co-Authored-By: Francisco Javier Arceo --------- Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/pr_integration_tests.yml | 4 ++++ .github/workflows/pr_local_integration_tests.yml | 4 ++++ .github/workflows/smoke_tests.yml | 7 ++++++- .github/workflows/unit_tests.yml | 7 ++++++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5a1b483b39e..923c0b0335b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -6,6 +6,10 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 8b2f8c13d2e..e6a9e3e8bde 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -7,6 +7,10 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: integration-test-python-local: diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 774d58d22b4..9a898dd4c54 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -1,6 +1,11 @@ name: smoke-tests -on: [pull_request] +on: + pull_request: + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: unit-test-python: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a8ddd397e30..6f46d129638 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -1,6 +1,11 @@ name: unit-tests -on: [pull_request] +on: + pull_request: + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: unit-test-python: runs-on: ${{ matrix.os }} From df74ee94fe8e029957d77d612d9db8f5198463c7 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 11 Dec 2024 15:44:47 -0500 Subject: [PATCH 31/90] =?UTF-8?q?chore:=20Refactoring=20release=20yamls=20?= =?UTF-8?q?to=20be=20more=20module=20and=20not=20use=20redund=E2=80=A6=20(?= =?UTF-8?q?#4826)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated workflows from master Signed-off-by: Francisco Javier Arceo * removing on push piece Signed-off-by: Francisco Javier Arceo * removing show semantic release Signed-off-by: Francisco Javier Arceo * remove linebreak Signed-off-by: Francisco Javier Arceo * updated all worfklows to use get_semantic_version.yml instead of reimplement it Signed-off-by: Francisco Javier Arceo * Updated order so if SDK fails to build images won't release Signed-off-by: Francisco Javier Arceo * updated publish to reference helm charts Signed-off-by: Francisco Javier Arceo * Updating publish python sdk to reference build wheels Signed-off-by: Francisco Javier Arceo * removed line break Signed-off-by: Francisco Javier Arceo * linted workflowsw Signed-off-by: Francisco Javier Arceo * updated Signed-off-by: Francisco Javier Arceo * added needs to publish docker images Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- .github/workflows/build_wheels.yml | 68 +----- .../get_semantic_release_version.yml | 84 +++++++ .github/workflows/publish.yml | 226 ++---------------- .github/workflows/publish_helm_charts.yml | 50 ++++ .github/workflows/publish_images.yml | 77 +----- .github/workflows/publish_java_sdk.yml | 58 +++++ .github/workflows/publish_python_sdk.yml | 41 ++++ .github/workflows/release.yml | 50 ++-- .github/workflows/show_semantic_release.yml | 38 --- 9 files changed, 299 insertions(+), 393 deletions(-) create mode 100644 .github/workflows/get_semantic_release_version.yml create mode 100644 .github/workflows/publish_helm_charts.yml create mode 100644 .github/workflows/publish_java_sdk.yml create mode 100644 .github/workflows/publish_python_sdk.yml delete mode 100644 .github/workflows/show_semantic_release.yml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 14ca70bcfc3..39713924111 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,8 +5,8 @@ name: build_wheels # Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly. on: workflow_dispatch: - tags: - - 'v*.*.*' + tags: + - 'v*.*.*' workflow_call: inputs: release_version: @@ -20,58 +20,10 @@ on: jobs: get-version: - runs-on: ubuntu-latest - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Get release version - id: get_release_version - run: | - if [[ -n "${{ inputs.release_version }}" ]]; then - echo "Using provided release version: ${{ inputs.release_version }}" - echo "::set-output name=release_version::${{ inputs.release_version }}" - else - echo "No release version provided. Falling back to GITHUB_REF." - echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" - fi - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - if [[ -n "${{ inputs.highest_semver_tag }}" ]]; then - echo "Using provided highest semver version: ${{ inputs.highest_semver_tag }}" - echo "::set-output name=highest_semver_tag::${{ inputs.highest_semver_tag }}" - else - echo "No release version provided. Falling back to infra/scripts/setup-common-functions.sh." - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - fi - fi - - name: Check output - id: check_output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} build-python-wheel: name: Build wheels @@ -141,7 +93,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server, feature-server-java, feature-transformation-server] + component: [ feature-server, feature-server-java, feature-transformation-server ] env: REGISTRY: feastdev steps: @@ -158,11 +110,11 @@ jobs: verify-python-wheels: runs-on: ${{ matrix.os }} - needs: [build-python-wheel, build-source-distribution, get-version] + needs: [ build-python-wheel, build-source-distribution, get-version ] strategy: matrix: - os: [ubuntu-latest, macos-13 ] - python-version: ["3.9", "3.10", "3.11"] + os: [ ubuntu-latest, macos-13 ] + python-version: [ "3.9", "3.10", "3.11" ] from-source: [ True, False ] env: # this script is for testing servers diff --git a/.github/workflows/get_semantic_release_version.yml b/.github/workflows/get_semantic_release_version.yml new file mode 100644 index 00000000000..c800810dced --- /dev/null +++ b/.github/workflows/get_semantic_release_version.yml @@ -0,0 +1,84 @@ +name: Get semantic release version + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GIT_AUTHOR_NAME: feast-ci-bot + GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co + GIT_COMMITTER_NAME: feast-ci-bot + GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co + outputs: + release_version: ${{ steps.get_release_version.outputs.release_version }} + version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + steps: + - uses: actions/checkout@v4 + - name: Get release version + id: get_release_version + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + echo "Using custom version: ${{ github.event.inputs.custom_version }}" + if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then + echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." + exit 1 + fi + echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" + elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then + echo "Using tag reference: ${GITHUB_REF#refs/tags/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" + else + echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" + fi + - name: Get release version without prefix + id: get_release_version_without_prefix + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ "${RELEASE_VERSION}" == v* ]]; then + echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" + else + echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" + fi + - name: Get highest semver + id: get_highest_semver + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" + else + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" + fi + fi + - name: Check output + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + run: | + echo $RELEASE_VERSION + echo $VERSION_WITHOUT_PREFIX + echo $HIGHEST_SEMVER_TAG \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fcd4a1b7201..eb81ca193cd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,212 +17,34 @@ on: jobs: get-version: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ github.event.inputs.token }} - GIT_AUTHOR_NAME: feast-ci-bot - GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co - GIT_COMMITTER_NAME: feast-ci-bot - GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - uses: actions/checkout@v4 - - name: Get release version - id: get_release_version - run: | - if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then - VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" - echo "Using custom version: ${{ github.event.inputs.custom_version }}" - if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then - echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." - exit 1 - fi - echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" - elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then - echo "Using tag reference: ${GITHUB_REF#refs/tags/}" - echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" - else - echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" - echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" - fi - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - if [[ "${RELEASE_VERSION}" == v* ]]; then - echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" - else - echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" - fi - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then - HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" - echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" - echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" - else - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" - fi - fi - - name: Check output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} - build-publish-docker-images: - runs-on: ubuntu-latest - needs: [get-version, publish-python-sdk] - strategy: - matrix: - component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator] - env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: feastdev - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - name: Use gcloud CLI - run: gcloud info - - run: gcloud auth configure-docker --quiet - - name: Build image - run: | - make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - - name: Push versioned images - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + publish-python-sdk: + uses: ./.github/workflows/publish_python_sdk.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest - docker push feastdev/${{ matrix.component }}:latest - fi + build-publish-docker-images: + uses: ./.github/workflows/publish_images.yaml + needs: [ get-version, publish-python-sdk ] + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} publish-helm-charts: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - needs: get-version - env: - HELM_VERSION: v3.8.0 - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - steps: - - uses: actions/checkout@v4 - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - run: gcloud auth configure-docker --quiet - - name: Remove previous Helm - run: sudo rm -rf $(which helm) - - name: Install Helm - run: ./infra/scripts/helm/install-helm.sh - - name: Validate Helm chart prior to publishing - run: ./infra/scripts/helm/validate-helm-chart-publish.sh - - name: Validate all version consistency - run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX - - name: Publish Helm charts - run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX - - build_wheels: - uses: ./.github/workflows/build_wheels.yml - needs: get-version + uses: ./.github/workflows/publish_helm_charts.yml + needs: [ get-version, publish-python-sdk ] with: - release_version: ${{ needs.get-version.outputs.release_version }} - highest_semver_tag: ${{ needs.get-version.outputs.highest_semver_tag }} - - publish-python-sdk: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - needs: [build_wheels] - steps: - - uses: actions/download-artifact@v4.1.7 - with: - name: python-wheels - path: dist - - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} publish-java-sdk: - if: github.repository == 'feast-dev/feast' - container: maven:3.6-jdk-11 - runs-on: ubuntu-latest - needs: get-version - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - architecture: 'x64' - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-it-maven- - - name: Publish java sdk - env: - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} - run: | - echo -n "$GPG_PUBLIC_KEY" > /root/public-key - echo -n "$GPG_PRIVATE_KEY" > /root/private-key - mkdir -p /root/.m2/ - echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml - infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root + uses: ./.github/workflows/publish_java_sdk.yml + needs: [ get-version, publish-python-sdk ] + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} diff --git a/.github/workflows/publish_helm_charts.yml b/.github/workflows/publish_helm_charts.yml new file mode 100644 index 00000000000..060bdb05d4e --- /dev/null +++ b/.github/workflows/publish_helm_charts.yml @@ -0,0 +1,50 @@ +name: publish images + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + publish-helm-charts: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + needs: get-version + env: + HELM_VERSION: v3.8.0 + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + steps: + - uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + - run: gcloud auth configure-docker --quiet + - name: Remove previous Helm + run: sudo rm -rf $(which helm) + - name: Install Helm + run: ./infra/scripts/helm/install-helm.sh + - name: Validate Helm chart prior to publishing + run: ./infra/scripts/helm/validate-helm-chart-publish.sh + - name: Validate all version consistency + run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX + - name: Publish Helm charts + run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX + diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index a80036cc9ff..26201aaa5c2 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -14,81 +14,18 @@ on: jobs: get-version: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ github.event.inputs.token }} - GIT_AUTHOR_NAME: feast-ci-bot - GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co - GIT_COMMITTER_NAME: feast-ci-bot - GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - uses: actions/checkout@v4 - - name: Get release version - id: get_release_version - run: | - if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then - VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" - echo "Using custom version: ${{ github.event.inputs.custom_version }}" - if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then - echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." - exit 1 - fi - echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" - elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then - echo "Using tag reference: ${GITHUB_REF#refs/tags/}" - echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" - else - echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" - echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" - fi - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - if [[ "${RELEASE_VERSION}" == v* ]]; then - echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" - else - echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" - fi - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then - HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" - echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" - echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" - else - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" - fi - fi - - name: Check output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} build-publish-docker-images: + if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest - needs: [get-version] + needs: [ get-version ] strategy: matrix: - component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator] + component: [ feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/.github/workflows/publish_java_sdk.yml b/.github/workflows/publish_java_sdk.yml new file mode 100644 index 00000000000..c158010995d --- /dev/null +++ b/.github/workflows/publish_java_sdk.yml @@ -0,0 +1,58 @@ +name: publish java sdk + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + publish-java-sdk: + if: github.repository == 'feast-dev/feast' + container: maven:3.6-jdk-11 + runs-on: ubuntu-latest + needs: [ get-version, publish-python-sdk ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: '11' + java-package: jdk + architecture: x64 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + architecture: 'x64' + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-it-maven- + - name: Publish java sdk + env: + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} + run: | + echo -n "$GPG_PUBLIC_KEY" > /root/public-key + echo -n "$GPG_PRIVATE_KEY" > /root/private-key + mkdir -p /root/.m2/ + echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml + infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root diff --git a/.github/workflows/publish_python_sdk.yml b/.github/workflows/publish_python_sdk.yml new file mode 100644 index 00000000000..1a9c111de74 --- /dev/null +++ b/.github/workflows/publish_python_sdk.yml @@ -0,0 +1,41 @@ +name: publish python sdk + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + build_wheels: + uses: ./.github/workflows/build_wheels.yml + needs: get-version + with: + release_version: ${{ needs.get-version.outputs.release_version }} + highest_semver_tag: ${{ needs.get-version.outputs.highest_semver_tag }} + + publish-python-sdk: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + needs: [ get-version, build_wheels ] + steps: + - uses: actions/download-artifact@v4.1.7 + with: + name: python-wheels + path: dist + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec7ffb29eda..f367bb33fe8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ on: type: boolean jobs: - + if: github.repository == 'feast-dev/feast' get_dry_release_versions: runs-on: ubuntu-latest env: @@ -98,7 +98,7 @@ jobs: make build-installer bundle publish-web-ui-npm: - needs: [validate_version_bumps, get_dry_release_versions] + needs: [ validate_version_bumps, get_dry_release_versions ] runs-on: ubuntu-latest env: # This publish is working using an NPM automation token to bypass 2FA @@ -121,7 +121,7 @@ jobs: run: yarn build:lib - name: Publish UI package working-directory: ./ui - if: github.event.inputs.dry_run == 'false' && github.event.inputs.publish_ui == 'true' + if: github.event.inputs.dry_run == 'false' && github.event.inputs.publish_ui == 'true' run: npm publish env: # This publish is working using an NPM automation token to bypass 2FA @@ -138,25 +138,25 @@ jobs: GIT_COMMITTER_NAME: feast-ci-bot GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version-file: './ui/.nvmrc' - - name: Set up Homebrew - id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master - - name: Setup Helm-docs - run: | - brew install norwoodj/tap/helm-docs - - name: Release (Dry Run) - if: github.event.inputs.dry_run == 'true' - run: | - npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run - - name: Release - if: github.event.inputs.dry_run == 'false' - run: | - npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version-file: './ui/.nvmrc' + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Setup Helm-docs + run: | + brew install norwoodj/tap/helm-docs + - name: Release (Dry Run) + if: github.event.inputs.dry_run == 'true' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run + - name: Release + if: github.event.inputs.dry_run == 'false' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release diff --git a/.github/workflows/show_semantic_release.yml b/.github/workflows/show_semantic_release.yml deleted file mode 100644 index f4aef6be54b..00000000000 --- a/.github/workflows/show_semantic_release.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: show semantic release versions - -on: - workflow_dispatch: - inputs: - token: - description: 'Personal Access Token' - required: true - default: "" - type: string - -jobs: - - get_dry_release_versions: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ github.event.inputs.token }} - outputs: - current_version: ${{ steps.get_versions.outputs.current_version }} - next_version: ${{ steps.get_versions.outputs.next_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: "lts/*" - - name: Release (Dry Run) - id: get_versions - run: | - CURRENT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep "associated with version " | sed -E 's/.* version//' | sed -E 's/ on.*//') - NEXT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep 'The next release version is' | sed -E 's/.* ([[:digit:].]+)$/\1/') - echo ::set-output name=current_version::$CURRENT_VERSION - echo ::set-output name=next_version::$NEXT_VERSION - echo "Current version is ${CURRENT_VERSION}" - echo "Next version is ${NEXT_VERSION}" From b97da6ca3a08e3f0fc35552dd7f0bd3b59083f35 Mon Sep 17 00:00:00 2001 From: Daniel Dowler <12484302+dandawg@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:43:35 -0700 Subject: [PATCH 32/90] feat: Add online/offline replica support (#4812) * added replica support to online/offline store services Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * Removed unneaded if statement Co-authored-by: Tommy Hughes IV Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * fixed missing bracket Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * added doc comments describing replicas Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * Replicas doc wording change Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> --------- Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> Co-authored-by: Tommy Hughes IV --- .../api/v1alpha1/featurestore_types.go | 20 ++- .../api/v1alpha1/zz_generated.deepcopy.go | 25 +++- .../crd/bases/feast.dev_featurestores.yaml | 24 ++++ infra/feast-operator/dist/install.yaml | 24 ++++ .../featurestore_controller_db_store_test.go | 3 +- .../featurestore_controller_ephemeral_test.go | 3 +- ...restore_controller_kubernetes_auth_test.go | 3 +- ...eaturestore_controller_objectstore_test.go | 3 +- .../featurestore_controller_oidc_auth_test.go | 3 +- .../featurestore_controller_pvc_test.go | 3 +- .../featurestore_controller_test.go | 132 ++++++++++++++++-- .../internal/controller/services/services.go | 17 ++- .../internal/controller/services/util.go | 11 +- 13 files changed, 244 insertions(+), 27 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 454c909f7d0..635912a1b5f 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -74,9 +74,9 @@ type FeatureStoreServices struct { // OfflineStore configures the deployed offline store service type OfflineStore struct { - ServiceConfigs `json:",inline"` - Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *OfflineTlsConfigs `json:"tls,omitempty"` + StoreServiceConfigs `json:",inline"` + Persistence *OfflineStorePersistence `json:"persistence,omitempty"` + TLS *OfflineTlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the offline store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -134,9 +134,9 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ // OnlineStore configures the deployed online store service type OnlineStore struct { - ServiceConfigs `json:",inline"` - Persistence *OnlineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + StoreServiceConfigs `json:",inline"` + Persistence *OnlineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the online store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -297,6 +297,14 @@ type DefaultConfigs struct { Image *string `json:"image,omitempty"` } +// StoreServiceConfigs k8s deployment settings +type StoreServiceConfigs struct { + // Replicas determines the number of pods for the feast service. + // When Replicas > 1, persistence is recommended. + Replicas *int32 `json:"replicas,omitempty"` + ServiceConfigs `json:",inline"` +} + // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 3f317c650e9..bccf9ec5378 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -273,7 +273,7 @@ func (in *LocalRegistryConfig) DeepCopy() *LocalRegistryConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { *out = *in - in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) + in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OfflineStorePersistence) @@ -397,7 +397,7 @@ func (in *OidcAuthz) DeepCopy() *OidcAuthz { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = *in - in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) + in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OnlineStorePersistence) @@ -737,6 +737,27 @@ func (in *ServiceHostnames) DeepCopy() *ServiceHostnames { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StoreServiceConfigs) DeepCopyInto(out *StoreServiceConfigs) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoreServiceConfigs. +func (in *StoreServiceConfigs) DeepCopy() *StoreServiceConfigs { + if in == nil { + return nil + } + out := new(StoreServiceConfigs) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 61b3e7adcf0..74f0fd059e3 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -369,6 +369,12 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -745,6 +751,12 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1614,6 +1626,12 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1997,6 +2015,12 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 5d56e786394..f40c5caebb7 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -377,6 +377,12 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -753,6 +759,12 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1622,6 +1634,12 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2005,6 +2023,12 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' + replicas: + description: |- + Replicas determines the number of pods for the feast service. + When Replicas > 1, persistence is recommended. + format: int32 + type: integer resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 377ee6bc512..0ee269bda17 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -127,6 +127,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Context("When deploying a resource with all db storage services", func() { const resourceName = "cr-name" var pullPolicy = corev1.PullAlways + var replicas = int32(1) ctx := context.Background() @@ -205,7 +206,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index 796de8e5260..a762faa5a21 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -48,6 +48,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { const resourceName = "services-ephemeral" const offlineType = "duckdb" var pullPolicy = corev1.PullAlways + var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -65,7 +66,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 4930f3fc590..57dd3a290df 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -48,6 +48,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Context("When deploying a resource with all ephemeral services and Kubernetes authorization", func() { const resourceName = "kubernetes-authorization" var pullPolicy = corev1.PullAlways + var replicas = int32(1) ctx := context.Background() @@ -62,7 +63,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index db07418c92b..f4a21a28f17 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -46,6 +46,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-object-store" var pullPolicy = corev1.PullAlways + var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -67,7 +68,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index c062a573df2..eb320c5bb39 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -49,6 +49,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { const resourceName = "oidc-authorization" const oidcSecretName = "oidc-secret" var pullPolicy = corev1.PullAlways + var replicas = int32(1) ctx := context.Background() @@ -73,7 +74,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index d0adc62c7c8..fe0caa38e63 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -50,6 +50,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-pvc" var pullPolicy = corev1.PullAlways + var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -77,7 +78,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 44c81eca59a..debd63300b2 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -404,6 +404,7 @@ var _ = Describe("FeatureStore Controller", func() { Context("When reconciling a resource with all services enabled", func() { const resourceName = "services" var pullPolicy = corev1.PullAlways + var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -419,7 +420,7 @@ var _ = Describe("FeatureStore Controller", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -870,6 +871,114 @@ var _ = Describe("FeatureStore Controller", func() { Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) + It("Should scale online/offline store service", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name}) + Expect(err).NotTo(HaveOccurred()) + labelSelector := labels.NewSelector().Add(*req) + listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector} + deployList := appsv1.DeploymentList{} + err = k8sClient.List(ctx, &deployList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(deployList.Items).To(HaveLen(3)) + + svcList := corev1.ServiceList{} + err = k8sClient.List(ctx, &svcList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(HaveLen(3)) + + cmList := corev1.ConfigMapList{} + err = k8sClient.List(ctx, &cmList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(cmList.Items).To(HaveLen(1)) + + feast := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + }, + } + + fsYamlStr := "" + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + Expect(err).NotTo(HaveOccurred()) + + // check online config + deploy_online := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, + deploy_online) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy_online.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_online.Name)) + Expect(deploy_online.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy_online.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(deploy_online.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) + Expect(deploy_online.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) + + // check offline config + deploy_offline := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, + deploy_offline) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy_offline.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_offline.Name)) + Expect(deploy_offline.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy_offline.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy_offline.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + + // change feast project and reconcile + // scale online replicas to 2 + resourceNew := resource.DeepCopy() + new_replicas := int32(2) + resourceNew.Spec.Services.OnlineStore.Replicas = &new_replicas + resourceNew.Spec.Services.OfflineStore.Replicas = &new_replicas + + err = k8sClient.Update(ctx, resourceNew) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, + deploy_online) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, + deploy_offline) + Expect(err).NotTo(HaveOccurred()) + + Expect(deploy_online.Spec.Replicas).To(Equal(&new_replicas)) + Expect(deploy_offline.Spec.Replicas).To(Equal(&new_replicas)) + }) + It("Should delete k8s objects owned by the FeatureStore CR", func() { By("changing which feast services are configured in the CR") controllerReconciler := &FeatureStoreReconciler{ @@ -1253,7 +1362,7 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, replicas int32, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -1264,14 +1373,17 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co Services: &feastdevv1alpha1.FeatureStoreServices{ OfflineStore: &feastdevv1alpha1.OfflineStore{}, OnlineStore: &feastdevv1alpha1.OnlineStore{ - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, + StoreServiceConfigs: feastdevv1alpha1.StoreServiceConfigs{ + Replicas: &replicas, + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, + }, }, }, }, diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 60aabebe024..0f18cc55224 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -293,7 +293,7 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F probeHandler := getProbeHandler(feastType, tls) deploy.Spec = appsv1.DeploymentSpec{ - Replicas: &DefaultReplicas, + Replicas: feast.getServiceReplicas(feastType), Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -485,6 +485,21 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } +func (feast *FeastServices) getServiceReplicas(feastType FeastServiceType) *int32 { + appliedServices := feast.Handler.FeatureStore.Status.Applied.Services + switch feastType { + case OfflineFeastType: + if feast.isOfflinStore() { + return appliedServices.OfflineStore.Replicas + } + case OnlineFeastType: + if feast.isOnlinStore() { + return appliedServices.OnlineStore.Replicas + } + } + return &DefaultReplicas +} + func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { services := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 85bd02e653a..631709d6ba0 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -122,7 +122,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) + setStoreServiceDefaultConfigs(&services.OfflineStore.StoreServiceConfigs) } if services.OnlineStore != nil { @@ -147,7 +147,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) + setStoreServiceDefaultConfigs(&services.OnlineStore.StoreServiceConfigs) } // overwrite status.applied with every reconcile applied.DeepCopyInto(&cr.Status.Applied) @@ -159,6 +159,13 @@ func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { } } +func setStoreServiceDefaultConfigs(storeServiceConfigs *feastdevv1alpha1.StoreServiceConfigs) { + if storeServiceConfigs.Replicas == nil { + storeServiceConfigs.Replicas = &DefaultReplicas + } + setServiceDefaultConfigs(&storeServiceConfigs.ServiceConfigs.DefaultConfigs) +} + func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1alpha1.ValidOfflineStoreFilePersistenceTypes, value) { return nil From c5ffa037cb030c64d6e25995199cf762cc0e9b2a Mon Sep 17 00:00:00 2001 From: Niklas von Maltzahn Date: Thu, 12 Dec 2024 12:23:19 +0200 Subject: [PATCH 33/90] feat: Implement `date_partition_column` for `SparkSource` (#4844) --- .../contrib/spark_offline_store/spark.py | 11 +- .../spark_offline_store/spark_source.py | 6 + .../contrib/spark_offline_store/test_spark.py | 121 ++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index aeb9e3cd68b..4b501886327 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -99,6 +99,8 @@ def pull_latest_from_table_or_query( fields_as_string = ", ".join(fields_with_aliases) aliases_as_string = ", ".join(aliases) + date_partition_column = data_source.date_partition_column + start_date_str = _format_datetime(start_date) end_date_str = _format_datetime(end_date) query = f""" @@ -109,7 +111,7 @@ def pull_latest_from_table_or_query( SELECT {fields_as_string}, ROW_NUMBER() OVER({partition_by_join_key_string} ORDER BY {timestamp_desc_string}) AS feast_row_ FROM {from_expression} t1 - WHERE {timestamp_field} BETWEEN TIMESTAMP('{start_date_str}') AND TIMESTAMP('{end_date_str}') + WHERE {timestamp_field} BETWEEN TIMESTAMP('{start_date_str}') AND TIMESTAMP('{end_date_str}'){" AND "+date_partition_column+" >= '"+start_date.strftime('%Y-%m-%d')+"' AND "+date_partition_column+" <= '"+end_date.strftime('%Y-%m-%d')+"' " if date_partition_column != "" and date_partition_column is not None else ''} ) t2 WHERE feast_row_ = 1 """ @@ -641,8 +643,15 @@ def _cast_data_frame( {% endfor %} FROM {{ featureview.table_subquery }} WHERE {{ featureview.timestamp_field }} <= '{{ featureview.max_event_timestamp }}' + {% if featureview.date_partition_column != "" and featureview.date_partition_column is not none %} + AND {{ featureview.date_partition_column }} <= '{{ featureview.max_event_timestamp[:10] }}' + {% endif %} + {% if featureview.ttl == 0 %}{% else %} AND {{ featureview.timestamp_field }} >= '{{ featureview.min_event_timestamp }}' + {% if featureview.date_partition_column != "" and featureview.date_partition_column is not none %} + AND {{ featureview.date_partition_column }} >= '{{ featureview.min_event_timestamp[:10] }}' + {% endif %} {% endif %} ), diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 209e3b87e8b..7ad331239ff 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -45,6 +45,7 @@ def __init__( tags: Optional[Dict[str, str]] = None, owner: Optional[str] = "", timestamp_field: Optional[str] = None, + date_partition_column: Optional[str] = None, ): """Creates a SparkSource object. @@ -64,6 +65,8 @@ def __init__( maintainer. timestamp_field: Event timestamp field used for point-in-time joins of feature values. + date_partition_column: The column to partition the data on for faster + retrieval. This is useful for large tables and will limit the number ofi """ # If no name, use the table as the default name. if name is None and table is None: @@ -77,6 +80,7 @@ def __init__( created_timestamp_column=created_timestamp_column, field_mapping=field_mapping, description=description, + date_partition_column=date_partition_column, tags=tags, owner=owner, ) @@ -135,6 +139,7 @@ def from_proto(data_source: DataSourceProto) -> Any: query=spark_options.query, path=spark_options.path, file_format=spark_options.file_format, + date_partition_column=data_source.date_partition_column, timestamp_field=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, description=data_source.description, @@ -148,6 +153,7 @@ def to_proto(self) -> DataSourceProto: type=DataSourceProto.BATCH_SPARK, data_source_class_type="feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource", field_mapping=self.field_mapping, + date_partition_column=self.date_partition_column, spark_options=self.spark_options.to_proto(), description=self.description, tags=self.tags, diff --git a/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py b/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py index b8f8cc42474..307ba4058c1 100644 --- a/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py +++ b/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py @@ -71,6 +71,68 @@ def test_pull_latest_from_table_with_nested_timestamp_or_query(mock_get_spark_se assert retrieval_job.query.strip() == expected_query.strip() +@patch( + "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" +) +def test_pull_latest_from_table_with_nested_timestamp_or_query_and_date_partition_column_set( + mock_get_spark_session, +): + mock_spark_session = MagicMock() + mock_get_spark_session.return_value = mock_spark_session + + test_repo_config = RepoConfig( + project="test_project", + registry="test_registry", + provider="local", + offline_store=SparkOfflineStoreConfig(type="spark"), + ) + + test_data_source = SparkSource( + name="test_nested_batch_source", + description="test_nested_batch_source", + table="offline_store_database_name.offline_store_table_name", + timestamp_field="nested_timestamp", + field_mapping={ + "event_header.event_published_datetime_utc": "nested_timestamp", + }, + date_partition_column="effective_date", + ) + + # Define the parameters for the method + join_key_columns = ["key1", "key2"] + feature_name_columns = ["feature1", "feature2"] + timestamp_field = "event_header.event_published_datetime_utc" + created_timestamp_column = "created_timestamp" + start_date = datetime(2021, 1, 1) + end_date = datetime(2021, 1, 2) + + # Call the method + retrieval_job = SparkOfflineStore.pull_latest_from_table_or_query( + config=test_repo_config, + data_source=test_data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + ) + + expected_query = """SELECT + key1, key2, feature1, feature2, nested_timestamp, created_timestamp + + FROM ( + SELECT key1, key2, feature1, feature2, event_header.event_published_datetime_utc AS nested_timestamp, created_timestamp, + ROW_NUMBER() OVER(PARTITION BY key1, key2 ORDER BY event_header.event_published_datetime_utc DESC, created_timestamp DESC) AS feast_row_ + FROM `offline_store_database_name`.`offline_store_table_name` t1 + WHERE event_header.event_published_datetime_utc BETWEEN TIMESTAMP('2021-01-01 00:00:00.000000') AND TIMESTAMP('2021-01-02 00:00:00.000000') AND effective_date >= '2021-01-01' AND effective_date <= '2021-01-02' + ) t2 + WHERE feast_row_ = 1""" # noqa: W293, W291 + + assert isinstance(retrieval_job, RetrievalJob) + assert retrieval_job.query.strip() == expected_query.strip() + + @patch( "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" ) @@ -127,3 +189,62 @@ def test_pull_latest_from_table_without_nested_timestamp_or_query( assert isinstance(retrieval_job, RetrievalJob) assert retrieval_job.query.strip() == expected_query.strip() + + +@patch( + "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" +) +def test_pull_latest_from_table_without_nested_timestamp_or_query_and_date_partition_column_set( + mock_get_spark_session, +): + mock_spark_session = MagicMock() + mock_get_spark_session.return_value = mock_spark_session + + test_repo_config = RepoConfig( + project="test_project", + registry="test_registry", + provider="local", + offline_store=SparkOfflineStoreConfig(type="spark"), + ) + + test_data_source = SparkSource( + name="test_batch_source", + description="test_nested_batch_source", + table="offline_store_database_name.offline_store_table_name", + timestamp_field="event_published_datetime_utc", + date_partition_column="effective_date", + ) + + # Define the parameters for the method + join_key_columns = ["key1", "key2"] + feature_name_columns = ["feature1", "feature2"] + timestamp_field = "event_published_datetime_utc" + created_timestamp_column = "created_timestamp" + start_date = datetime(2021, 1, 1) + end_date = datetime(2021, 1, 2) + + # Call the method + retrieval_job = SparkOfflineStore.pull_latest_from_table_or_query( + config=test_repo_config, + data_source=test_data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + ) + + expected_query = """SELECT + key1, key2, feature1, feature2, event_published_datetime_utc, created_timestamp + + FROM ( + SELECT key1, key2, feature1, feature2, event_published_datetime_utc, created_timestamp, + ROW_NUMBER() OVER(PARTITION BY key1, key2 ORDER BY event_published_datetime_utc DESC, created_timestamp DESC) AS feast_row_ + FROM `offline_store_database_name`.`offline_store_table_name` t1 + WHERE event_published_datetime_utc BETWEEN TIMESTAMP('2021-01-01 00:00:00.000000') AND TIMESTAMP('2021-01-02 00:00:00.000000') AND effective_date >= '2021-01-01' AND effective_date <= '2021-01-02' + ) t2 + WHERE feast_row_ = 1""" # noqa: W293, W291 + + assert isinstance(retrieval_job, RetrievalJob) + assert retrieval_job.query.strip() == expected_query.strip() From b4768a81b94352de037dc305df309fcf06fd2973 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 12 Dec 2024 09:50:23 -0500 Subject: [PATCH 34/90] fix: Fix release.yml (#4845) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f367bb33fe8..79b845b101a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,8 @@ on: type: boolean jobs: - if: github.repository == 'feast-dev/feast' get_dry_release_versions: + if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ github.event.inputs.token }} From 88a92cf793c63fd8133bea44a317f6aece073346 Mon Sep 17 00:00:00 2001 From: Elliot Scribner Date: Thu, 12 Dec 2024 06:51:34 -0800 Subject: [PATCH 35/90] chore: Add Template for Couchbase Online Store (#4836) Add Template for Couchbase Online Store Signed-off-by: Elliot Scribner --- sdk/python/feast/cli.py | 1 + .../feast/templates/couchbase/__init__.py | 0 .../couchbase/feature_repo/__init__.py | 0 .../couchbase/feature_repo/feature_store.yaml | 11 +++++ .../feast/templates/couchbase/gitignore | 45 +++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 sdk/python/feast/templates/couchbase/__init__.py create mode 100644 sdk/python/feast/templates/couchbase/feature_repo/__init__.py create mode 100644 sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml create mode 100644 sdk/python/feast/templates/couchbase/gitignore diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index a02013b11f9..15b592119ca 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -865,6 +865,7 @@ def materialize_incremental_command(ctx: click.Context, end_ts: str, views: List "cassandra", "hazelcast", "ikv", + "couchbase", ], case_sensitive=False, ), diff --git a/sdk/python/feast/templates/couchbase/__init__.py b/sdk/python/feast/templates/couchbase/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/couchbase/feature_repo/__init__.py b/sdk/python/feast/templates/couchbase/feature_repo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..bc21e44defd --- /dev/null +++ b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml @@ -0,0 +1,11 @@ +project: my_project +registry: /path/to/registry.db +provider: local +online_store: + type: couchbase + connection_string: COUCHBASE_CONNECTION_STRING # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console + user: COUCHBASE_USER # Couchbase username from database access credentials + password: COUCHBASE_PASSWORD # Couchbase password from database access credentials + bucket_name: COUCHBASE_BUCKET_NAME # Couchbase bucket name, defaults to feast + kv_port: COUCHBASE_KV_PORT # Couchbase key-value port, defaults to 11210. Required if custom ports are used. +entity_key_serialization_version: 2 diff --git a/sdk/python/feast/templates/couchbase/gitignore b/sdk/python/feast/templates/couchbase/gitignore new file mode 100644 index 00000000000..e86277f60f4 --- /dev/null +++ b/sdk/python/feast/templates/couchbase/gitignore @@ -0,0 +1,45 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*.pyo +*.pyd + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +*.egg-info/ +dist/ +build/ +.venv + +# Pytest +.cache +*.cover +*.log +.coverage +nosetests.xml +coverage.xml +*.hypothesis/ +*.pytest_cache/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IDEs and Editors +.vscode/ +.idea/ +*.swp +*.swo +*.sublime-workspace +*.sublime-project + +# OS generated files +.DS_Store +Thumbs.db From 8320e23eb85cc419ef8aa0fdc07efa81857e0345 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:45:31 -0500 Subject: [PATCH 36/90] feat: Removing the tls_verify_client flag from feast cli for offline server. (#4842) * Removing the tls_verify_client flag for offline server from feast application code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fixing lint errors. formatted the code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/feast/cli.py | 12 +----------- sdk/python/feast/feature_store.py | 5 +---- sdk/python/feast/offline_server.py | 5 +---- .../feature_repos/universal/data_sources/file.py | 3 --- 4 files changed, 3 insertions(+), 22 deletions(-) diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 15b592119ca..ccfcd1471cf 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -1133,15 +1133,6 @@ def serve_registry_command( show_default=False, help="path to TLS certificate public key. You need to pass --key as well to start server in TLS mode", ) -@click.option( - "--verify_client", - "-v", - "tls_verify_client", - type=click.BOOL, - default="True", - show_default=True, - help="Verify the client or not for the TLS client certificate.", -) @click.pass_context def serve_offline_command( ctx: click.Context, @@ -1149,7 +1140,6 @@ def serve_offline_command( port: int, tls_key_path: str, tls_cert_path: str, - tls_verify_client: bool, ): """Start a remote server locally on a given host, port.""" if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path): @@ -1158,7 +1148,7 @@ def serve_offline_command( ) store = create_feature_store(ctx) - store.serve_offline(host, port, tls_key_path, tls_cert_path, tls_verify_client) + store.serve_offline(host, port, tls_key_path, tls_cert_path) @cli.command("validate") diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 79a0d752efb..44975902018 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1964,14 +1964,11 @@ def serve_offline( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ) -> None: """Start offline server locally on a given port.""" from feast import offline_server - offline_server.start_server( - self, host, port, tls_key_path, tls_cert_path, tls_verify_client - ) + offline_server.start_server(self, host, port, tls_key_path, tls_cert_path) def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 8774dea8aed..1b714a45c7e 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -45,7 +45,6 @@ def __init__( location: str, host: str = "localhost", tls_certificates: List = [], - verify_client=False, **kwargs, ): super(OfflineServer, self).__init__( @@ -54,7 +53,7 @@ def __init__( str_to_auth_manager_type(store.config.auth_config.type) ), tls_certificates=tls_certificates, - verify_client=verify_client, + verify_client=False, # this is needed for when we don't need mTLS **kwargs, ) self._location = location @@ -568,7 +567,6 @@ def start_server( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ): _init_auth_manager(store) @@ -591,7 +589,6 @@ def start_server( location=location, host=host, tls_certificates=tls_certificates, - verify_client=tls_verify_client, ) try: logger.info(f"Offline store server serving at: {location}") diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index dc716f45e1e..fbfb418278e 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -452,9 +452,6 @@ def setup(self, registry: RegistryConfig): str(tls_key_path), "--cert", str(self.tls_cert_path), - # This is needed for the self-signed certificate, disabled verify_client for integration tests. - "--verify_client", - str(False), ] self.proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL From 79fa247026dd95e75a19308d437997310d061b35 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Mon, 16 Dec 2024 13:46:52 -0600 Subject: [PATCH 37/90] fix: Remove verifyClient TLS offlineStore option from the Operator (#4847) remove verifyClient TLS option Signed-off-by: Tommy Hughes --- .../api/v1alpha1/featurestore_types.go | 9 +------- .../api/v1alpha1/zz_generated.deepcopy.go | 23 +------------------ .../crd/bases/feast.dev_featurestores.yaml | 19 +++++---------- infra/feast-operator/dist/install.yaml | 19 +++++---------- .../featurestore_controller_tls_test.go | 14 ++++------- .../controller/services/repo_config.go | 5 ++-- .../internal/controller/services/services.go | 14 ++--------- .../internal/controller/services/tls.go | 20 +++++----------- .../internal/controller/services/tls_test.go | 5 ---- 9 files changed, 29 insertions(+), 99 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 635912a1b5f..84b4d8e841b 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -76,20 +76,13 @@ type FeatureStoreServices struct { type OfflineStore struct { StoreServiceConfigs `json:",inline"` Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *OfflineTlsConfigs `json:"tls,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the offline store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical LogLevel string `json:"logLevel,omitempty"` } -// OfflineTlsConfigs configures server TLS for the offline feast service. in an openshift cluster, this is configured by default using service serving certificates. -type OfflineTlsConfigs struct { - TlsConfigs `json:",inline"` - // verify the client TLS certificate. - VerifyClient *bool `json:"verifyClient,omitempty"` -} - // OfflineStorePersistence configures the persistence settings for the offline store service // +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store." type OfflineStorePersistence struct { diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index bccf9ec5378..6cba8e59234 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -281,7 +281,7 @@ func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { } if in.TLS != nil { in, out := &in.TLS, &out.TLS - *out = new(OfflineTlsConfigs) + *out = new(TlsConfigs) (*in).DeepCopyInto(*out) } } @@ -357,27 +357,6 @@ func (in *OfflineStorePersistence) DeepCopy() *OfflineStorePersistence { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OfflineTlsConfigs) DeepCopyInto(out *OfflineTlsConfigs) { - *out = *in - in.TlsConfigs.DeepCopyInto(&out.TlsConfigs) - if in.VerifyClient != nil { - in, out := &in.VerifyClient, &out.VerifyClient - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OfflineTlsConfigs. -func (in *OfflineTlsConfigs) DeepCopy() *OfflineTlsConfigs { - if in == nil { - return nil - } - out := new(OfflineTlsConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OidcAuthz) DeepCopyInto(out *OidcAuthz) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 74f0fd059e3..7fbd38ed31b 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -432,9 +432,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -464,9 +464,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1690,10 +1687,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1723,9 +1719,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index f40c5caebb7..73abc3717b8 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -440,9 +440,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -472,9 +472,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1698,10 +1695,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1731,9 +1727,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index 45cda317409..c191dae3329 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -56,7 +56,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { } featurestore := &feastdevv1alpha1.FeatureStore{} localRef := corev1.LocalObjectReference{Name: "test"} - tlsConfigs := feastdevv1alpha1.TlsConfigs{ + tlsConfigs := &feastdevv1alpha1.TlsConfigs{ SecretRef: &localRef, } BeforeEach(func() { @@ -72,16 +72,14 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { FeastProject: feastProject, Services: &feastdevv1alpha1.FeatureStoreServices{ OnlineStore: &feastdevv1alpha1.OnlineStore{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Local: &feastdevv1alpha1.LocalRegistryConfig{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, }, }, @@ -396,9 +394,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { }, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index c70996ab867..5433e99acfd 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -248,9 +248,8 @@ func getClientRepoConfig( Host: strings.Split(status.ServiceHostnames.OfflineStore, ":")[0], Port: HttpPort, } - if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS != nil && - (&appliedServices.OfflineStore.TLS.TlsConfigs).IsTLS() { - clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.TlsConfigs.SecretKeyNames.TlsCrt + if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS.IsTLS() { + clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.SecretKeyNames.TlsCrt clientRepoConfig.OfflineStore.Port = HttpsPort clientRepoConfig.OfflineStore.Scheme = HttpsScheme } diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 0f18cc55224..f85597e648c 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -373,13 +373,6 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st } deploySettings.Args = append(deploySettings.Args, []string{"-p", strconv.Itoa(int(targetPort))}...) - if feastType == OfflineFeastType { - if tls.IsTLS() && feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient != nil { - deploySettings.Args = append(deploySettings.Args, - []string{"--verify_client", strconv.FormatBool(*feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient)}...) - } - } - // Combine base command, options, and arguments feastCommand := append([]string{baseCommand}, options...) feastCommand = append(feastCommand, deploySettings.Args...) @@ -549,11 +542,8 @@ func (feast *FeastServices) setServiceHostnames() error { domain := svcDomain + ":" if feast.isOfflinStore() { objMeta := feast.GetObjectMeta(OfflineFeastType) - port := strconv.Itoa(HttpPort) - if feast.offlineTls() { - port = strconv.Itoa(HttpsPort) - } - feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + port + feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS) } if feast.isOnlinStore() { objMeta := feast.GetObjectMeta(OnlineFeastType) diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index c92c4d8de23..a52cc707eb3 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -29,7 +29,7 @@ func (feast *FeastServices) setTlsDefaults() error { } appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tlsDefaults(&appliedServices.OfflineStore.TLS.TlsConfigs) + tlsDefaults(appliedServices.OfflineStore.TLS) } if feast.isOnlinStore() { tlsDefaults(appliedServices.OnlineStore.TLS) @@ -43,11 +43,9 @@ func (feast *FeastServices) setTlsDefaults() error { func (feast *FeastServices) setOpenshiftTls() error { appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.offlineOpenshiftTls() { - appliedServices.OfflineStore.TLS = &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: feastdevv1alpha1.TlsConfigs{ - SecretRef: &corev1.LocalObjectReference{ - Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, - }, + appliedServices.OfflineStore.TLS = &feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{ + Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, }, } } @@ -103,8 +101,8 @@ func (feast *FeastServices) getTlsConfigs(feastType FeastServiceType) (tls *feas appliedServices := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { case OfflineFeastType: - if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tls = &appliedServices.OfflineStore.TLS.TlsConfigs + if feast.isOfflinStore() { + tls = appliedServices.OfflineStore.TLS } case OnlineFeastType: if feast.isOnlinStore() { @@ -154,12 +152,6 @@ func (feast *FeastServices) remoteRegistryOpenshiftTls() (bool, error) { return false, nil } -func (feast *FeastServices) offlineTls() bool { - return feast.isOfflinStore() && - feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS != nil && - (&feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.TlsConfigs).IsTLS() -} - func (feast *FeastServices) localRegistryTls() bool { return localRegistryTls(feast.Handler.FeatureStore) } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 2a66d8a4fdd..17d23dcf72a 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -58,7 +58,6 @@ var _ = Describe("TLS Config", func() { Expect(tls.IsTLS()).To(BeFalse()) Expect(getPortStr(tls)).To(Equal("80")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -87,7 +86,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -127,7 +125,6 @@ var _ = Describe("TLS Config", func() { Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) Expect(tls.IsTLS()).To(BeTrue()) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) @@ -189,7 +186,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -238,7 +234,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("80")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) From 49171bd53fb8bfc325eb7167cac8cae18a28bd63 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 17 Dec 2024 14:46:01 -0500 Subject: [PATCH 38/90] feat: Adding packages for Milvus Online Store (#4854) Signed-off-by: Francisco Javier Arceo --- .../milvus_online_store/__init__.py | 0 .../milvus_repo_configuration.py | 12 +++++++ .../requirements/py3.10-ci-requirements.txt | 23 ++++++++++-- .../requirements/py3.10-requirements.txt | 2 +- .../requirements/py3.11-ci-requirements.txt | 23 ++++++++++-- .../requirements/py3.11-requirements.txt | 2 +- .../requirements/py3.9-ci-requirements.txt | 23 ++++++++++-- .../requirements/py3.9-requirements.txt | 2 +- .../universal/online_store/milvus.py | 35 +++++++++++++++++++ setup.py | 4 +++ 10 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py b/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py new file mode 100644 index 00000000000..8e8402862cb --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.milvus import ( + MilvusOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="milvus", online_store_creator=MilvusOnlineStoreCreator + ), +] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 54a64f5b1ce..f2b48d73624 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -180,6 +180,8 @@ elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -275,6 +277,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -443,13 +446,17 @@ markupsafe==3.0.2 # nbconvert # werkzeug marshmallow==3.23.1 - # via great-expectations + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -548,6 +555,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -614,6 +622,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -658,7 +667,7 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi @@ -679,6 +688,8 @@ pyjwt[crypto]==2.10.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 @@ -738,7 +749,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -839,6 +852,7 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -939,6 +953,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -1018,6 +1033,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 9a087b4a8eb..63a887e1aac 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -95,7 +95,7 @@ pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 43637fd2067..a9dceac08c9 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -178,6 +178,8 @@ elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus execnet==2.1.1 # via pytest-xdist executing==2.1.0 @@ -268,6 +270,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -434,13 +437,17 @@ markupsafe==3.0.2 # nbconvert # werkzeug marshmallow==3.23.1 - # via great-expectations + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -539,6 +546,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -605,6 +613,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -649,7 +658,7 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi @@ -670,6 +679,8 @@ pyjwt[crypto]==2.10.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 @@ -729,7 +740,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -830,6 +843,7 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -920,6 +934,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -994,6 +1009,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 8f776fdc457..42f89ecb6a5 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -93,7 +93,7 @@ pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3deb441827c..556e709c20a 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -182,6 +182,8 @@ elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -277,6 +279,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -452,13 +455,17 @@ markupsafe==3.0.2 # nbconvert # werkzeug marshmallow==3.23.1 - # via great-expectations + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -556,6 +563,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -622,6 +630,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -666,7 +675,7 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi @@ -687,6 +696,8 @@ pyjwt[crypto]==2.10.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 @@ -746,7 +757,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -847,6 +860,7 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -947,6 +961,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -1028,6 +1043,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==1.26.20 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 8c9fc036433..db7113dc2c2 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -97,7 +97,7 @@ pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.2 # via # feast (setup.py) # fastapi diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py new file mode 100644 index 00000000000..8ffee04c12f --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -0,0 +1,35 @@ +from typing import Any, Dict + +from testcontainers.milvus import MilvusContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MilvusOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.fixed_port = 19530 + self.container = MilvusContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( + self.fixed_port + ) + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + # Wait for Milvus server to be ready + host = "localhost" + port = self.container.get_exposed_port(self.fixed_port) + return { + "type": "milvus", + "host": host, + "port": int(port), + "index_type": "IVF_FLAT", + "metric_type": "L2", + "embedding_dim": 2, + "vector_enabled": True, + "nlist": 1, + } + + def teardown(self): + self.container.stop() diff --git a/setup.py b/setup.py index 815d1b23229..7593be4c376 100644 --- a/setup.py +++ b/setup.py @@ -156,6 +156,8 @@ GO_REQUIRED = ["cffi~=1.15.0"] +MILVUS_REQUIRED = ["pymilvus"] + CI_REQUIRED = ( [ "build", @@ -226,6 +228,7 @@ + OPENTELEMETRY + FAISS_REQUIRED + QDRANT_REQUIRED + + MILVUS_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -355,6 +358,7 @@ def run(self): "faiss": FAISS_REQUIRED, "qdrant": QDRANT_REQUIRED, "go": GO_REQUIRED, + "milvus": MILVUS_REQUIRED, }, include_package_data=True, license="Apache", From 9887b90dc809cfce7924e5505d173c448b62e20b Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 17 Dec 2024 20:59:01 -0500 Subject: [PATCH 39/90] chore: Update sphinx docs (#4856) Signed-off-by: Francisco Javier Arceo --- ...a.online_stores.cassandra_online_store.rst | 29 +++++++++++++++ ...a.online_stores.couchbase_online_store.rst | 29 +++++++++++++++ ...line_stores.elasticsearch_online_store.rst | 29 +++++++++++++++ ...a.online_stores.hazelcast_online_store.rst | 29 +++++++++++++++ ...infra.online_stores.hbase_online_store.rst | 29 +++++++++++++++ ...t.infra.online_stores.ikv_online_store.rst | 21 +++++++++++ ...nfra.online_stores.milvus_online_store.rst | 21 +++++++++++ ...infra.online_stores.mysql_online_store.rst | 29 +++++++++++++++ ...ra.online_stores.postgres_online_store.rst | 37 +++++++++++++++++++ ...nfra.online_stores.qdrant_online_store.rst | 29 +++++++++++++++ 10 files changed, 282 insertions(+) create mode 100644 sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst create mode 100644 sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst diff --git a/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst new file mode 100644 index 00000000000..7c5c3d371a7 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.cassandra\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst new file mode 100644 index 00000000000..29d51304928 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.couchbase\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.couchbase\_online\_store.couchbase module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.couchbase\_online\_store.couchbase\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst new file mode 100644 index 00000000000..d470e3301d0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.elasticsearch\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch\_repo\_configuration module +------------------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst new file mode 100644 index 00000000000..9cb565ca132 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hazelcast\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst new file mode 100644 index 00000000000..50ad80e0a9e --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hbase\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.hbase\_online\_store.hbase module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hbase\_online\_store.hbase\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst new file mode 100644 index 00000000000..391af17024f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.ikv\_online\_store package +===================================================== + +Submodules +---------- + +feast.infra.online\_stores.ikv\_online\_store.ikv module +-------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store.ikv + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst new file mode 100644 index 00000000000..ee9faa55dc0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.milvus\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.milvus\_online\_store.milvus\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store.milvus_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst new file mode 100644 index 00000000000..b1a9ea4f802 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.mysql\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.mysql\_online\_store.mysql module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.mysql\_online\_store.mysql\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst new file mode 100644 index 00000000000..9dfd200a4e1 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst @@ -0,0 +1,37 @@ +feast.infra.online\_stores.postgres\_online\_store package +========================================================== + +Submodules +---------- + +feast.infra.online\_stores.postgres\_online\_store.pgvector\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.pgvector_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres module +------------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst new file mode 100644 index 00000000000..5c210d4124d --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.qdrant\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.qdrant\_online\_store.qdrant module +-------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.qdrant\_online\_store.qdrant\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store + :members: + :undoc-members: + :show-inheritance: From 739eaa78e6d995ee0750292d2f8d81886a3f9829 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 18 Dec 2024 08:26:59 -0500 Subject: [PATCH 40/90] feat: Adding vector_search parameter to fields (#4855) * feat: Adding vector_search parameter to fields Signed-off-by: Francisco Javier Arceo * updated field to handle linter and updated sphinx docs Signed-off-by: Francisco Javier Arceo * Have to remove the equality test for the new fields...for now we're going to ignore them so it is backwards compatible Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- protos/feast/core/Feature.proto | 7 ++++++- .../docs/source/feast.infra.online_stores.rst | 19 ++++++++++++++++- sdk/python/docs/source/feast.infra.rst | 8 +++++++ sdk/python/feast/field.py | 21 +++++++++++++++++++ .../feast/protos/feast/core/DataSource_pb2.py | 4 +++- .../protos/feast/core/DataSource_pb2.pyi | 15 +++++++++++++ .../feast/protos/feast/core/Entity_pb2.py | 4 +++- .../feast/protos/feast/core/Entity_pb2.pyi | 15 +++++++++++++ .../protos/feast/core/FeatureService_pb2.py | 4 +++- .../protos/feast/core/FeatureService_pb2.pyi | 15 +++++++++++++ .../protos/feast/core/FeatureView_pb2.py | 4 +++- .../protos/feast/core/FeatureView_pb2.pyi | 15 +++++++++++++ .../feast/protos/feast/core/Feature_pb2.py | 8 +++---- .../feast/protos/feast/core/Feature_pb2.pyi | 10 ++++++++- .../feast/core/OnDemandFeatureView_pb2.py | 4 +++- .../feast/core/OnDemandFeatureView_pb2.pyi | 15 +++++++++++++ .../feast/registry/RegistryServer_pb2.py | 5 +++-- .../protos/feast/serving/GrpcServer_pb2.py | 5 +++-- 18 files changed, 162 insertions(+), 16 deletions(-) diff --git a/protos/feast/core/Feature.proto b/protos/feast/core/Feature.proto index 882de47eb9c..8a56d67905a 100644 --- a/protos/feast/core/Feature.proto +++ b/protos/feast/core/Feature.proto @@ -35,6 +35,11 @@ message FeatureSpecV2 { map tags = 3; // Description of the feature. - string description = 4; + + // Field indicating the vector will be indexed for vector similarity search + bool vector_index = 5; + + // Metric used for vector similarity search. + string vector_search_metric = 6; } diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst index ea714e45c5b..c07c7e0c279 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -7,7 +7,16 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.online_stores + feast.infra.online_stores.cassandra_online_store + feast.infra.online_stores.couchbase_online_store + feast.infra.online_stores.elasticsearch_online_store + feast.infra.online_stores.hazelcast_online_store + feast.infra.online_stores.hbase_online_store + feast.infra.online_stores.ikv_online_store + feast.infra.online_stores.milvus_online_store + feast.infra.online_stores.mysql_online_store + feast.infra.online_stores.postgres_online_store + feast.infra.online_stores.qdrant_online_store Submodules ---------- @@ -36,6 +45,14 @@ feast.infra.online\_stores.dynamodb module :undoc-members: :show-inheritance: +feast.infra.online\_stores.faiss\_online\_store module +------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.faiss_online_store + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.helpers module ----------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index b0046a2719e..791a4ace832 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -51,6 +51,14 @@ feast.infra.provider module :undoc-members: :show-inheritance: +feast.infra.supported\_async\_methods module +-------------------------------------------- + +.. automodule:: feast.infra.supported_async_methods + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index a41dcf5d5e6..fda1fbffe54 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -32,12 +32,16 @@ class Field: dtype: The type of the field, such as string or float. description: A human-readable description. tags: User-defined metadata in dictionary form. + vector_index: If set to True the field will be indexed for vector similarity search. + vector_search_metric: The metric used for vector similarity search. """ name: str dtype: FeastType description: str tags: Dict[str, str] + vector_index: bool + vector_search_metric: Optional[str] def __init__( self, @@ -46,6 +50,8 @@ def __init__( dtype: FeastType, description: str = "", tags: Optional[Dict[str, str]] = None, + vector_index: bool = False, + vector_search_metric: Optional[str] = None, ): """ Creates a Field object. @@ -55,11 +61,15 @@ def __init__( dtype: The type of the field, such as string or float. description (optional): A human-readable description. tags (optional): User-defined metadata in dictionary form. + vector_index (optional): If set to True the field will be indexed for vector similarity search. + vector_search_metric (optional): The metric used for vector similarity search. """ self.name = name self.dtype = dtype self.description = description self.tags = tags or {} + self.vector_index = vector_index + self.vector_search_metric = vector_search_metric def __eq__(self, other): if type(self) != type(other): @@ -70,6 +80,8 @@ def __eq__(self, other): or self.dtype != other.dtype or self.description != other.description or self.tags != other.tags + # or self.vector_index != other.vector_index + # or self.vector_search_metric != other.vector_search_metric ): return False return True @@ -87,6 +99,8 @@ def __repr__(self): f" dtype={self.dtype!r},\n" f" description={self.description!r},\n" f" tags={self.tags!r}\n" + f" vector_index={self.vector_index!r}\n" + f" vector_search_metric={self.vector_search_metric!r}\n" f")" ) @@ -96,11 +110,14 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" value_type = self.dtype.to_value_type() + vector_search_metric = self.vector_search_metric or "" return FieldProto( name=self.name, value_type=value_type.value, description=self.description, tags=self.tags, + vector_index=self.vector_index, + vector_search_metric=vector_search_metric, ) @classmethod @@ -112,11 +129,15 @@ def from_proto(cls, field_proto: FieldProto): field_proto: FieldProto protobuf object """ value_type = ValueType(field_proto.value_type) + vector_search_metric = getattr(field_proto, "vector_search_metric", "") + vector_index = getattr(field_proto, "vector_index", False) return cls( name=field_proto.name, dtype=from_value_type(value_type=value_type), tags=dict(field_proto.tags), description=field_proto.description, + vector_index=vector_index, + vector_search_metric=vector_search_metric, ) @classmethod diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.py b/sdk/python/feast/protos/feast/core/DataSource_pb2.py index b58c33a3830..68bee8d7609 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.py +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.py @@ -19,7 +19,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\x42T\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\"=\n\x0e\x44\x61taSourceList\x12+\n\x0b\x64\x61tasources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSourceBT\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -69,4 +69,6 @@ _globals['_DATASOURCE_PUSHOPTIONS']._serialized_end=2801 _globals['_DATASOURCE_SOURCETYPE']._serialized_start=2804 _globals['_DATASOURCE_SOURCETYPE']._serialized_end=3052 + _globals['_DATASOURCELIST']._serialized_start=3071 + _globals['_DATASOURCELIST']._serialized_end=3132 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi index 94336638e19..aadec3fad4c 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi @@ -557,3 +557,18 @@ class DataSource(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["options", b"options"]) -> typing_extensions.Literal["file_options", "bigquery_options", "kafka_options", "kinesis_options", "redshift_options", "request_data_options", "custom_options", "snowflake_options", "push_options", "spark_options", "trino_options", "athena_options"] | None: ... global___DataSource = DataSource + +class DataSourceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATASOURCES_FIELD_NUMBER: builtins.int + @property + def datasources(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___DataSource]: ... + def __init__( + self, + *, + datasources: collections.abc.Iterable[global___DataSource] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["datasources", b"datasources"]) -> None: ... + +global___DataSourceList = DataSourceList diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.py b/sdk/python/feast/protos/feast/core/Entity_pb2.py index 5a192854cab..2b3e7806736 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.py +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"2\n\nEntityList\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.EntityBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,4 +34,6 @@ _globals['_ENTITYSPECV2_TAGSENTRY']._serialized_end=429 _globals['_ENTITYMETA']._serialized_start=431 _globals['_ENTITYMETA']._serialized_end=558 + _globals['_ENTITYLIST']._serialized_start=560 + _globals['_ENTITYLIST']._serialized_end=610 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi index 732b3e10326..025817edfee 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi @@ -128,3 +128,18 @@ class EntityMeta(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["created_timestamp", b"created_timestamp", "last_updated_timestamp", b"last_updated_timestamp"]) -> None: ... global___EntityMeta = EntityMeta + +class EntityList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ENTITIES_FIELD_NUMBER: builtins.int + @property + def entities(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Entity]: ... + def __init__( + self, + *, + entities: collections.abc.Iterable[global___Entity] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["entities", b"entities"]) -> None: ... + +global___EntityList = EntityList diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py index cf6ac46ac54..642d5b010f9 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py @@ -16,7 +16,7 @@ from feast.protos.feast.core import FeatureViewProjection_pb2 as feast_dot_core_dot_FeatureViewProjection__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stinationBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stination\"I\n\x12\x46\x65\x61tureServiceList\x12\x33\n\x0f\x66\x65\x61tureservices\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureServiceBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -52,4 +52,6 @@ _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_end=1571 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_start=1526 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_end=1571 + _globals['_FEATURESERVICELIST']._serialized_start=1588 + _globals['_FEATURESERVICELIST']._serialized_end=1661 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi index b3305b72df9..0b1c0baa871 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi @@ -264,3 +264,18 @@ class LoggingConfig(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["destination", b"destination"]) -> typing_extensions.Literal["file_destination", "bigquery_destination", "redshift_destination", "snowflake_destination", "custom_destination", "athena_destination"] | None: ... global___LoggingConfig = LoggingConfig + +class FeatureServiceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATURESERVICES_FIELD_NUMBER: builtins.int + @property + def featureservices(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureService]: ... + def __init__( + self, + *, + featureservices: collections.abc.Iterable[global___FeatureService] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureservices", b"featureservices"]) -> None: ... + +global___FeatureServiceList = FeatureServiceList diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py index f1480593d9a..80d04c1ec3f 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py @@ -18,7 +18,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"@\n\x0f\x46\x65\x61tureViewList\x12-\n\x0c\x66\x65\x61tureviews\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureViewBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -38,4 +38,6 @@ _globals['_FEATUREVIEWMETA']._serialized_end=918 _globals['_MATERIALIZATIONINTERVAL']._serialized_start=920 _globals['_MATERIALIZATIONINTERVAL']._serialized_end=1039 + _globals['_FEATUREVIEWLIST']._serialized_start=1041 + _globals['_FEATUREVIEWLIST']._serialized_end=1105 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi index e1d4e2dfee8..57158fc2c6c 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi @@ -192,3 +192,18 @@ class MaterializationInterval(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["end_time", b"end_time", "start_time", b"start_time"]) -> None: ... global___MaterializationInterval = MaterializationInterval + +class FeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def featureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureView]: ... + def __init__( + self, + *, + featureviews: collections.abc.Iterable[global___FeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureviews", b"featureviews"]) -> None: ... + +global___FeatureViewList = FeatureViewList diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.py b/sdk/python/feast/protos/feast/core/Feature_pb2.py index dd7c6008ef1..6b1081fe811 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.py +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.py @@ -15,7 +15,7 @@ from feast.protos.feast.types import Value_pb2 as feast_dot_types_dot_Value__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xc3\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xf7\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x14\n\x0cvector_index\x18\x05 \x01(\x08\x12\x1c\n\x14vector_search_metric\x18\x06 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,7 +26,7 @@ _globals['_FEATURESPECV2_TAGSENTRY']._options = None _globals['_FEATURESPECV2_TAGSENTRY']._serialized_options = b'8\001' _globals['_FEATURESPECV2']._serialized_start=66 - _globals['_FEATURESPECV2']._serialized_end=261 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=218 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=261 + _globals['_FEATURESPECV2']._serialized_end=313 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=270 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=313 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi index f4235b0965b..451f1aa61ce 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi @@ -53,6 +53,8 @@ class FeatureSpecV2(google.protobuf.message.Message): VALUE_TYPE_FIELD_NUMBER: builtins.int TAGS_FIELD_NUMBER: builtins.int DESCRIPTION_FIELD_NUMBER: builtins.int + VECTOR_INDEX_FIELD_NUMBER: builtins.int + VECTOR_SEARCH_METRIC_FIELD_NUMBER: builtins.int name: builtins.str """Name of the feature. Not updatable.""" value_type: feast.types.Value_pb2.ValueType.Enum.ValueType @@ -62,6 +64,10 @@ class FeatureSpecV2(google.protobuf.message.Message): """Tags for user defined metadata on a feature""" description: builtins.str """Description of the feature.""" + vector_index: builtins.bool + """Field indicating the vector will be indexed for vector similarity search""" + vector_search_metric: builtins.str + """Metric used for vector similarity search.""" def __init__( self, *, @@ -69,7 +75,9 @@ class FeatureSpecV2(google.protobuf.message.Message): value_type: feast.types.Value_pb2.ValueType.Enum.ValueType = ..., tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., description: builtins.str = ..., + vector_index: builtins.bool = ..., + vector_search_metric: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type", "vector_index", b"vector_index", "vector_search_metric", b"vector_search_metric"]) -> None: ... global___FeatureSpecV2 = FeatureSpecV2 diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py index 020515a6b89..926b54df288 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py @@ -20,7 +20,7 @@ from feast.protos.feast.core import Transformation_pb2 as feast_dot_core_dot_Transformation__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\x42]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\"X\n\x17OnDemandFeatureViewList\x12=\n\x14ondemandfeatureviews\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureViewB]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -50,4 +50,6 @@ _globals['_ONDEMANDSOURCE']._serialized_end=1371 _globals['_USERDEFINEDFUNCTION']._serialized_start=1373 _globals['_USERDEFINEDFUNCTION']._serialized_end=1445 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_start=1447 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_end=1535 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi index 3380779c97e..c9fca2f550d 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi @@ -233,3 +233,18 @@ class UserDefinedFunction(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "body_text", b"body_text", "name", b"name"]) -> None: ... global___UserDefinedFunction = UserDefinedFunction + +class OnDemandFeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ONDEMANDFEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def ondemandfeatureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___OnDemandFeatureView]: ... + def __init__( + self, + *, + ondemandfeatureviews: collections.abc.Iterable[global___OnDemandFeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["ondemandfeatureviews", b"ondemandfeatureviews"]) -> None: ... + +global___OnDemandFeatureViewList = OnDemandFeatureViewList diff --git a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py index e0cae3da4b7..2d5f7b020ab 100644 --- a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py +++ b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py @@ -28,13 +28,14 @@ from feast.protos.feast.core import Project_pb2 as feast_dot_core_dot_Project__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x42\x35Z3github.com/feast-dev/feast/go/protos/feast/registryb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.registry.RegistryServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z3github.com/feast-dev/feast/go/protos/feast/registry' _globals['_LISTENTITIESREQUEST_TAGSENTRY']._options = None _globals['_LISTENTITIESREQUEST_TAGSENTRY']._serialized_options = b'8\001' _globals['_LISTDATASOURCESREQUEST_TAGSENTRY']._options = None diff --git a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py index 8e40630cfff..ce4db37a658 100644 --- a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py +++ b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py @@ -15,13 +15,14 @@ from feast.protos.feast.serving import ServingService_pb2 as feast_dot_serving_dot_ServingService__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseB4Z2github.com/feast-dev/feast/go/protos/feast/servingb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.serving.GrpcServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z2github.com/feast-dev/feast/go/protos/feast/serving' _globals['_PUSHREQUEST_FEATURESENTRY']._options = None _globals['_PUSHREQUEST_FEATURESENTRY']._serialized_options = b'8\001' _globals['_WRITETOONLINESTOREREQUEST_FEATURESENTRY']._options = None From 132ce2a6c9e3ff8544680d5237e9e1523d988d7e Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Wed, 18 Dec 2024 12:55:45 -0500 Subject: [PATCH 41/90] feat: Loading the CA trusted store certificate into Feast to verify the public certificate. (#4852) * Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the lint error. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Trying to fix the online store test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Formatted the python to fix lint errors. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the unit test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the unit test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * removing unnecessary cli args. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Now configuring the SSL ca store configurations on the feast client side rather than on the server side. And also fixing the integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Renamed the remote registry is_tls_mode variable to is_tls. Changed the offline store TLS setting decision from cert to scheme. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Adding the existing trust store certificates to the newly created trust store. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * combining the default system ca store with the custom one to fix the integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Final clean up and adding documentation. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Incorporating the code review comments from Francisco. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../starting-feast-servers-tls-mode.md | 5 + sdk/python/feast/cli.py | 1 - sdk/python/feast/feature_store.py | 13 +- .../feast/infra/offline_stores/remote.py | 2 +- sdk/python/feast/infra/registry/remote.py | 37 +++- sdk/python/feast/ssl_ca_trust_store_setup.py | 22 +++ sdk/python/tests/conftest.py | 31 +++- .../universal/data_sources/file.py | 2 +- .../online_store/test_remote_online_store.py | 39 ++-- .../auth/server/test_auth_registry_server.py | 5 +- .../tests/utils/auth_permissions_util.py | 25 ++- .../generate_self_signed_certifcate_util.py | 79 -------- .../tests/utils/ssl_certifcates_util.py | 174 ++++++++++++++++++ 13 files changed, 320 insertions(+), 115 deletions(-) create mode 100644 sdk/python/feast/ssl_ca_trust_store_setup.py delete mode 100644 sdk/python/tests/utils/generate_self_signed_certifcate_util.py create mode 100644 sdk/python/tests/utils/ssl_certifcates_util.py diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md index e1ddbc08be5..a868e17cf96 100644 --- a/docs/how-to-guides/starting-feast-servers-tls-mode.md +++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md @@ -189,3 +189,8 @@ INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on https://0.0.0.0:8888 (Press CTRL+C to quit) ``` + + +## Adding public key to CA trust store and configuring the feast to use the trust store. +You can pass the public key for SSL verification using the `cert` parameter, however, it is sometimes difficult to maintain individual certificates and pass them individually. +The alternative recommendation is to add the public certificate to CA trust store and set the path as an environment variable (e.g., `FEAST_CA_CERT_FILE_PATH`). Feast will use the trust store path in the `FEAST_CA_CERT_FILE_PATH` environment variable. \ No newline at end of file diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index ccfcd1471cf..165677a843a 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -982,7 +982,6 @@ def serve_command( raise click.BadParameter( "Please pass --cert and --key args to start the feature server in TLS mode." ) - store = create_feature_store(ctx) store.serve( diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 44975902018..edbd060e106 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -86,6 +86,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.ssl_ca_trust_store_setup import configure_ca_trust_store_env_variables from feast.stream_feature_view import StreamFeatureView from feast.utils import _utc_now @@ -129,6 +130,8 @@ def __init__( if fs_yaml_file is not None and config is not None: raise ValueError("You cannot specify both fs_yaml_file and config.") + configure_ca_trust_store_env_variables() + if repo_path: self.repo_path = Path(repo_path) else: @@ -1949,13 +1952,19 @@ def serve_ui( ) def serve_registry( - self, port: int, tls_key_path: str = "", tls_cert_path: str = "" + self, + port: int, + tls_key_path: str = "", + tls_cert_path: str = "", ) -> None: """Start registry server locally on a given port.""" from feast import registry_server registry_server.start_server( - self, port=port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path + self, + port=port, + tls_key_path=tls_key_path, + tls_cert_path=tls_cert_path, ) def serve_offline( diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py index 6f26e06c6ba..d11fb4673db 100644 --- a/sdk/python/feast/infra/offline_stores/remote.py +++ b/sdk/python/feast/infra/offline_stores/remote.py @@ -74,7 +74,7 @@ def build_arrow_flight_client( scheme: str, host: str, port, auth_config: AuthConfig, cert: str = "" ): arrow_scheme = "grpc+tcp" - if cert: + if scheme == "https": logger.info( "Scheme is https so going to connect offline server in SSL(TLS) mode." ) diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index 6cc80d5dad1..590c0454b73 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from pathlib import Path from typing import List, Optional, Union @@ -59,6 +60,12 @@ class RemoteRegistryConfig(RegistryConfig): """ str: Path to the public certificate when the registry server starts in TLS(SSL) mode. This may be needed if the registry server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`. If registry_type is 'remote', then this configuration is needed to connect to remote registry server in TLS mode. If the remote registry started in non-tls mode then this configuration is not needed.""" + is_tls: bool = False + """ bool: Set to `True` if you plan to connect to a registry server running in TLS (SSL) mode. + If you intend to add the public certificate to the trust store instead of passing it via the `cert` parameter, this field must be set to `True`. + If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `true` is mandatory. + """ + class RemoteRegistry(BaseRegistry): def __init__( @@ -70,20 +77,32 @@ def __init__( ): self.auth_config = auth_config assert isinstance(registry_config, RemoteRegistryConfig) - if registry_config.cert: - with open(registry_config.cert, "rb") as cert_file: - trusted_certs = cert_file.read() - tls_credentials = grpc.ssl_channel_credentials( - root_certificates=trusted_certs - ) - self.channel = grpc.secure_channel(registry_config.path, tls_credentials) - else: - self.channel = grpc.insecure_channel(registry_config.path) + self.channel = self._create_grpc_channel(registry_config) auth_header_interceptor = GrpcClientAuthHeaderInterceptor(auth_config) self.channel = grpc.intercept_channel(self.channel, auth_header_interceptor) self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel) + def _create_grpc_channel(self, registry_config): + assert isinstance(registry_config, RemoteRegistryConfig) + if registry_config.cert or registry_config.is_tls: + cafile = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE") + if not cafile and not registry_config.cert: + raise EnvironmentError( + "SSL_CERT_FILE or REQUESTS_CA_BUNDLE environment variable must be set to use secure TLS or set the cert parameter in feature_Store.yaml file under remote registry configuration." + ) + with open( + registry_config.cert if registry_config.cert else cafile, "rb" + ) as cert_file: + trusted_certs = cert_file.read() + tls_credentials = grpc.ssl_channel_credentials( + root_certificates=trusted_certs + ) + return grpc.secure_channel(registry_config.path, tls_credentials) + else: + # Create an insecure gRPC channel + return grpc.insecure_channel(registry_config.path) + def close(self): if self.channel: self.channel.close() diff --git a/sdk/python/feast/ssl_ca_trust_store_setup.py b/sdk/python/feast/ssl_ca_trust_store_setup.py new file mode 100644 index 00000000000..72e84132187 --- /dev/null +++ b/sdk/python/feast/ssl_ca_trust_store_setup.py @@ -0,0 +1,22 @@ +import logging +import os + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def configure_ca_trust_store_env_variables(): + """ + configures the environment variable so that other libraries or servers refer to the TLS ca file path. + :param ca_file_path: + :return: + """ + if ( + "FEAST_CA_CERT_FILE_PATH" in os.environ + and os.environ["FEAST_CA_CERT_FILE_PATH"] + ): + logger.info( + f"Feast CA Cert file path found in environment variable FEAST_CA_CERT_FILE_PATH={os.environ['FEAST_CA_CERT_FILE_PATH']}. Going to refer this path." + ) + os.environ["SSL_CERT_FILE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] + os.environ["REQUESTS_CA_BUNDLE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 24c8f40f742..6e5f1e14870 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -57,8 +57,12 @@ location, ) from tests.utils.auth_permissions_util import default_store -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import ( + combine_trust_stores, + create_ca_trust_store, + generate_self_signed_cert, +) logger = logging.getLogger(__name__) @@ -514,17 +518,36 @@ def auth_config(request, is_integration_test): return auth_configuration -@pytest.fixture(params=[True, False], scope="module") +@pytest.fixture(scope="module") def tls_mode(request): - is_tls_mode = request.param + is_tls_mode = request.param[0] + output_combined_truststore_path = "" if is_tls_mode: certificates_path = tempfile.mkdtemp() tls_key_path = os.path.join(certificates_path, "key.pem") tls_cert_path = os.path.join(certificates_path, "cert.pem") + generate_self_signed_cert(cert_path=tls_cert_path, key_path=tls_key_path) + is_ca_trust_store_set = request.param[1] + if is_ca_trust_store_set: + # Paths + feast_ca_trust_store_path = os.path.join( + certificates_path, "feast_trust_store.pem" + ) + create_ca_trust_store( + public_key_path=tls_cert_path, + private_key_path=tls_key_path, + output_trust_store_path=feast_ca_trust_store_path, + ) + + # Combine trust stores + output_combined_path = os.path.join( + certificates_path, "combined_trust_store.pem" + ) + combine_trust_stores(feast_ca_trust_store_path, output_combined_path) else: tls_key_path = "" tls_cert_path = "" - return is_tls_mode, tls_key_path, tls_cert_path + return is_tls_mode, tls_key_path, tls_cert_path, output_combined_truststore_path diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index fbfb418278e..1d33402e012 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -34,8 +34,8 @@ DataSourceCreator, ) from tests.utils.auth_permissions_util import include_auth_config -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import generate_self_signed_cert logger = logging.getLogger(__name__) diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 10f1180d8e6..285253dfaaf 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -22,6 +22,9 @@ @pytest.mark.integration +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_remote_online_store_read(auth_config, tls_mode): with ( tempfile.TemporaryDirectory() as remote_server_tmp_dir, @@ -56,13 +59,13 @@ def test_remote_online_store_read(auth_config, tls_mode): ) ) assert None not in (server_store, server_url, registry_path) - _, _, tls_cert_path = tls_mode + client_store = _create_remote_client_feature_store( temp_dir=remote_client_tmp_dir, server_registry_path=str(registry_path), feature_server_url=server_url, auth_config=auth_config, - tls_cert_path=tls_cert_path, + tls_mode=tls_mode, ) assert client_store is not None _assert_non_existing_entity_feature_views_entity( @@ -172,7 +175,7 @@ def _create_server_store_spin_feature_server( ): store = default_store(str(temp_dir), auth_config, permissions_list) feast_server_port = free_port() - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, ca_trust_store_path = tls_mode server_url = next( start_feature_server( @@ -180,6 +183,7 @@ def _create_server_store_spin_feature_server( server_port=feast_server_port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path, + ca_trust_store_path=ca_trust_store_path, ) ) if is_tls_mode: @@ -203,20 +207,33 @@ def _create_remote_client_feature_store( server_registry_path: str, feature_server_url: str, auth_config: str, - tls_cert_path: str = "", + tls_mode, ) -> FeatureStore: project_name = "REMOTE_ONLINE_CLIENT_PROJECT" runner = CliRunner() result = runner.run(["init", project_name], cwd=temp_dir) assert result.returncode == 0 repo_path = os.path.join(temp_dir, project_name, "feature_repo") - _overwrite_remote_client_feature_store_yaml( - repo_path=str(repo_path), - registry_path=server_registry_path, - feature_server_url=feature_server_url, - auth_config=auth_config, - tls_cert_path=tls_cert_path, - ) + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode + if is_tls_mode and not ca_trust_store_path: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + tls_cert_path=tls_cert_path, + ) + else: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + ) + + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path return FeatureStore(repo_path=repo_path) diff --git a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py index 25c5fe3eb8c..0395f995410 100644 --- a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py +++ b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py @@ -44,7 +44,7 @@ def start_registry_server( assertpy.assert_that(server_port).is_not_equal_to(0) - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, tls_ca_file_path = tls_mode if is_tls_mode: print(f"Starting Registry in TLS mode at {server_port}") server = start_server( @@ -74,6 +74,9 @@ def start_registry_server( server.stop(grace=None) # Teardown server +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_registry_apis( auth_config, tls_mode, diff --git a/sdk/python/tests/utils/auth_permissions_util.py b/sdk/python/tests/utils/auth_permissions_util.py index 6f0a3c8eeac..8a1e7b7c4d7 100644 --- a/sdk/python/tests/utils/auth_permissions_util.py +++ b/sdk/python/tests/utils/auth_permissions_util.py @@ -60,6 +60,7 @@ def start_feature_server( metrics: bool = False, tls_key_path: str = "", tls_cert_path: str = "", + ca_trust_store_path: str = "", ): host = "0.0.0.0" cmd = [ @@ -127,18 +128,30 @@ def start_feature_server( def get_remote_registry_store(server_port, feature_store, tls_mode): - is_tls_mode, _, tls_cert_path = tls_mode + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode if is_tls_mode: - registry_config = RemoteRegistryConfig( - registry_type="remote", - path=f"localhost:{server_port}", - cert=tls_cert_path, - ) + if ca_trust_store_path: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + ) + else: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + cert=tls_cert_path, + ) else: registry_config = RemoteRegistryConfig( registry_type="remote", path=f"localhost:{server_port}" ) + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path + store = FeatureStore( config=RepoConfig( project=PROJECT_NAME, diff --git a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py b/sdk/python/tests/utils/generate_self_signed_certifcate_util.py deleted file mode 100644 index 559ee18cde7..00000000000 --- a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py +++ /dev/null @@ -1,79 +0,0 @@ -import ipaddress -import logging -from datetime import datetime, timedelta - -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID - -logger = logging.getLogger(__name__) - - -def generate_self_signed_cert( - cert_path="cert.pem", key_path="key.pem", common_name="localhost" -): - """ - Generate a self-signed certificate and save it to the specified paths. - - :param cert_path: Path to save the certificate (PEM format) - :param key_path: Path to save the private key (PEM format) - :param common_name: Common name (CN) for the certificate, defaults to 'localhost' - """ - # Generate private key - key = rsa.generate_private_key( - public_exponent=65537, key_size=2048, backend=default_backend() - ) - - # Create a self-signed certificate - subject = issuer = x509.Name( - [ - x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), - x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), - x509.NameAttribute(NameOID.COMMON_NAME, common_name), - ] - ) - - # Define the certificate's Subject Alternative Names (SANs) - alt_names = [ - x509.DNSName("localhost"), # Hostname - x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP - x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) - ] - san = x509.SubjectAlternativeName(alt_names) - - certificate = ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after( - # Certificate valid for 1 year - datetime.utcnow() + timedelta(days=365) - ) - .add_extension(san, critical=False) - .sign(key, hashes.SHA256(), default_backend()) - ) - - # Write the private key to a file - with open(key_path, "wb") as f: - f.write( - key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - - # Write the certificate to a file - with open(cert_path, "wb") as f: - f.write(certificate.public_bytes(serialization.Encoding.PEM)) - - logger.info( - f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." - ) diff --git a/sdk/python/tests/utils/ssl_certifcates_util.py b/sdk/python/tests/utils/ssl_certifcates_util.py new file mode 100644 index 00000000000..53a56e04f3d --- /dev/null +++ b/sdk/python/tests/utils/ssl_certifcates_util.py @@ -0,0 +1,174 @@ +import ipaddress +import logging +import os +import shutil +from datetime import datetime, timedelta + +import certifi +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509 import load_pem_x509_certificate +from cryptography.x509.oid import NameOID + +logger = logging.getLogger(__name__) + + +def generate_self_signed_cert( + cert_path="cert.pem", key_path="key.pem", common_name="localhost" +): + """ + Generate a self-signed certificate and save it to the specified paths. + + :param cert_path: Path to save the certificate (PEM format) + :param key_path: Path to save the private key (PEM format) + :param common_name: Common name (CN) for the certificate, defaults to 'localhost' + """ + # Generate private key + key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + + # Create a self-signed certificate + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), + ] + ) + + # Define the certificate's Subject Alternative Names (SANs) + alt_names = [ + x509.DNSName("localhost"), # Hostname + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP + x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) + ] + san = x509.SubjectAlternativeName(alt_names) + + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after( + # Certificate valid for 1 year + datetime.utcnow() + timedelta(days=365) + ) + .add_extension(san, critical=False) + .sign(key, hashes.SHA256(), default_backend()) + ) + + # Write the private key to a file + with open(key_path, "wb") as f: + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + # Write the certificate to a file + with open(cert_path, "wb") as f: + f.write(certificate.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." + ) + + +def create_ca_trust_store( + public_key_path: str, private_key_path: str, output_trust_store_path: str +): + """ + Create a new CA trust store as a copy of the existing one (if available), + and add the provided public certificate to it. + + :param public_key_path: Path to the public certificate (e.g., PEM file). + :param private_key_path: Path to the private key (optional, to verify signing authority). + :param output_trust_store_path: Path to save the new trust store. + """ + try: + # Step 1: Identify the existing trust store (if available via environment variables) + existing_trust_store = os.environ.get("SSL_CERT_FILE") or os.environ.get( + "REQUESTS_CA_BUNDLE" + ) + + # Step 2: Copy the existing trust store to the new location (if it exists) + if existing_trust_store and os.path.exists(existing_trust_store): + shutil.copy(existing_trust_store, output_trust_store_path) + logger.info( + f"Copied existing trust store from {existing_trust_store} to {output_trust_store_path}" + ) + else: + # Log the creation of a new trust store (without opening a file unnecessarily) + logger.info( + f"No existing trust store found. Creating a new trust store at {output_trust_store_path}" + ) + + # Step 3: Load and validate the public certificate + with open(public_key_path, "rb") as pub_file: + public_cert_data = pub_file.read() + public_cert = load_pem_x509_certificate( + public_cert_data, backend=default_backend() + ) + + # Verify the private key matches (optional, adds validation) + if private_key_path: + with open(private_key_path, "rb") as priv_file: + private_key_data = priv_file.read() + private_key = serialization.load_pem_private_key( + private_key_data, password=None, backend=default_backend() + ) + # Check the public/private key match + if ( + private_key.public_key().public_numbers() + != public_cert.public_key().public_numbers() + ): + raise ValueError( + "Public certificate does not match the private key." + ) + + # Step 4: Add the public certificate to the new trust store + with open(output_trust_store_path, "ab") as trust_store_file: + trust_store_file.write(public_cert.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Trust store created/updated successfully at: {output_trust_store_path}" + ) + + except Exception as e: + logger.error(f"Error creating CA trust store: {e}") + + +def combine_trust_stores(custom_cert_path: str, output_combined_path: str): + """ + Combine the default certifi CA bundle with a custom certificate file. + + :param custom_cert_path: Path to the custom certificate PEM file. + :param output_combined_path: Path where the combined CA bundle will be saved. + """ + try: + # Get the default certifi CA bundle + certifi_ca_bundle = certifi.where() + + with open(output_combined_path, "wb") as combined_file: + # Write the default CA bundle + with open(certifi_ca_bundle, "rb") as default_file: + combined_file.write(default_file.read()) + + # Append the custom certificates + with open(custom_cert_path, "rb") as custom_file: + combined_file.write(custom_file.read()) + + logger.info(f"Combined trust store created at: {output_combined_path}") + + except Exception as e: + logger.error(f"Error combining trust stores: {e}") + raise e From 31afd99c0969002fe04982e40cf7a857960f7abf Mon Sep 17 00:00:00 2001 From: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> Date: Wed, 18 Dec 2024 22:34:46 +0100 Subject: [PATCH 42/90] fix: Updated python-helm-demo example to use MinIO instead of GS (#4691) * Updated python-helm-demo example to use MinIO instead of GS Signed-off-by: Daniele Martinoli * Update examples/python-helm-demo/README.md Co-authored-by: Francisco Arceo Signed-off-by: Daniele Martinoli * Adding explicit wait to container to validate CI failures Signed-off-by: Daniele Martinoli * restored original conftest Signed-off-by: Daniele Martinoli --------- Signed-off-by: Daniele Martinoli Co-authored-by: Francisco Arceo --- examples/python-helm-demo/README.md | 159 +++++++++++++----- .../data/driver_stats_with_string.parquet | Bin 35310 -> 35693 bytes .../feature_repo/feature_store.yaml | 10 -- .../feature_repo/feature_store.yaml.template | 9 + examples/python-helm-demo/minio-dev.yaml | 128 ++++++++++++++ examples/python-helm-demo/minio.env | 7 + .../online_feature_store.yaml.template | 7 + .../python-helm-demo/test/feature_store.yaml | 7 + .../test_python_fetch.py | 10 +- 9 files changed, 287 insertions(+), 50 deletions(-) delete mode 100644 examples/python-helm-demo/feature_repo/feature_store.yaml create mode 100644 examples/python-helm-demo/feature_repo/feature_store.yaml.template create mode 100644 examples/python-helm-demo/minio-dev.yaml create mode 100644 examples/python-helm-demo/minio.env create mode 100644 examples/python-helm-demo/online_feature_store.yaml.template create mode 100644 examples/python-helm-demo/test/feature_store.yaml rename examples/python-helm-demo/{feature_repo => test}/test_python_fetch.py (73%) diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md index 90469e746d4..078550ae392 100644 --- a/examples/python-helm-demo/README.md +++ b/examples/python-helm-demo/README.md @@ -3,87 +3,168 @@ For this tutorial, we set up Feast with Redis. -We use the Feast CLI to register and materialize features, and then retrieving via a Feast Python feature server deployed in Kubernetes +We use the Feast CLI to register and materialize features from the current machine, and then retrieving via a +Feast Python feature server deployed in Kubernetes ## First, let's set up a Redis cluster 1. Start minikube (`minikube start`) -2. Use helm to install a default Redis cluster +1. Use helm to install a default Redis cluster ```bash helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install my-redis bitnami/redis ``` ![](redis-screenshot.png) -3. Port forward Redis so we can materialize features to it +1. Port forward Redis so we can materialize features to it ```bash kubectl port-forward --namespace default svc/my-redis-master 6379:6379 ``` -4. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. +1. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. ```bash export REDIS_PASSWORD=$(kubectl get secret --namespace default my-redis -o jsonpath="{.data.redis-password}" | base64 --decode) echo $REDIS_PASSWORD ``` +## Then, let's set up a MinIO S3 store +Manifests have been taken from [Deploy Minio in your project](https://ai-on-openshift.io/tools-and-applications/minio/minio/#deploy-minio-in-your-project). + +1. Deploy MinIO instance: + ``` + kubectl apply -f minio-dev.yaml + ``` + +1. Forward the UI port: + ```console + kubectl port-forward svc/minio-service 9090:9090 + ``` +1. Login to (localhost:9090)[http://localhost:9090] as `minio`/`minio123` and create bucket called `feast-demo`. +1. Stop previous port forwarding and forward the API port instead: + ```console + kubectl port-forward svc/minio-service 9000:9000 + ``` + ## Next, we setup a local Feast repo -1. Install Feast with Redis dependencies `pip install "feast[redis]"` -2. Make a bucket in GCS (or S3) -3. The feature repo is already setup here, so you just need to swap in your GCS bucket and Redis credentials. - We need to modify the `feature_store.yaml`, which has two fields for you to replace: +1. Install Feast with Redis dependencies `pip install "feast[redis,aws]"` +1. The feature repo is already setup here, so you just need to swap in your Redis credentials. + We need to modify the `feature_store.yaml`, which has one field for you to replace: + ```console + sed "s/_REDIS_PASSWORD_/${REDIS_PASSWORD}/" feature_repo/feature_store.yaml.template > feature_repo/feature_store.yaml + cat feature_repo/feature_store.yaml + ``` + + Example repo: ```yaml - registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db + registry: s3://localhost:9000/feast-demo/registry.db project: feast_python_demo - provider: gcp + provider: local online_store: type: redis - # Note: this would normally be using instance URL's to access Redis - connection_string: localhost:6379,password=[YOUR PASSWORD] + connection_string: localhost:6379,password=**** offline_store: type: file entity_key_serialization_version: 2 ``` -4. Run `feast apply` from within the `feature_repo` directory to apply your local features to the remote registry - - Note: you may need to authenticate to gcloud first with `gcloud auth login` -5. Materialize features to the online store: +1. To run `feast apply` from the current machine we need to define the AWS credentials to connect the MinIO S3 store, which +are defined in [minio.env](./minio.env): + ```console + source minio.env + cd feature_repo + feast apply + ``` +1. Let's validate the setup by running some queries + ```console + feast entities list + feast feature-views list + ``` +1. Materialize features to the online store: ```bash + cd feature_repo CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") feast materialize-incremental $CURRENT_TIME ``` ## Now let's setup the Feast Server -1. Add the gcp-auth addon to mount GCP credentials: - ```bash - minikube addons enable gcp-auth - ``` -2. Add Feast's Python/Go feature server chart repo +1. Add Feast's Python feature server chart repo ```bash helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -3. For this tutorial, because we don't have a direct hosted endpoint into Redis, we need to change `feature_store.yaml` to talk to the Kubernetes Redis service - ```bash - sed -i '' 's/localhost:6379/my-redis-master:6379/g' feature_store.yaml - ``` -4. Install the Feast helm chart: `helm install feast-release feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml)` - > **Dev instructions**: if you're changing the java logic or chart, you can do - 1. `eval $(minikube docker-env)` - 2. `make build-feature-server-dev` - 3. `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` -5. (Optional): check logs of the server to make sure it’s working +1. For this tutorial, we'll use a predefined configuration where we just needs to inject the Redis service password: + ```console + sed "s/_REDIS_PASSWORD_/$REDIS_PASSWORD/" online_feature_store.yaml.template > online_feature_store.yaml + cat online_feature_store.yaml + ``` + As you see, the connection points to `my-redis-master:6379` instead of `localhost:6379`. + +1. Install the Feast helm chart: + ```console + helm upgrade --install feast-online feast-charts/feast-feature-server \ + --set fullnameOverride=online-server --set feast_mode=online \ + --set feature_store_yaml_base64=$(base64 -i 'online_feature_store.yaml') + ``` +1. Patch the deployment to include MinIO settings: + ```console + kubectl patch deployment online-server --type='json' -p='[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_ACCESS_KEY_ID", + "value": "minio" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_SECRET_ACCESS_KEY", + "value": "minio123" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_DEFAULT_REGION", + "value": "default" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "FEAST_S3_ENDPOINT_URL", + "value": "http://minio-service:9000" + } + } + ]' + kubectl wait --for=condition=available deployment/online-server --timeout=2m + ``` +1. (Optional): check logs of the server to make sure it’s working ```bash - kubectl logs svc/feast-release-feast-feature-server + kubectl logs svc/online-server ``` -6. Port forward to expose the grpc endpoint: +1. Port forward to expose the grpc endpoint: ```bash - kubectl port-forward svc/feast-release-feast-feature-server 6566:80 + kubectl port-forward svc/online-server 6566:80 ``` -7. Run test fetches for online features:8. - - First: change back the Redis connection string to allow localhost connections to Redis +1. Run test fetches for online features:8. ```bash - sed -i '' 's/my-redis-master:6379/localhost:6379/g' feature_store.yaml + source minio.env + cd test + python test_python_fetch.py ``` - - Then run the included fetch script, which fetches both via the HTTP endpoint and for comparison, via the Python SDK - ```bash - python test_python_fetch.py + + Output example: + ```console + --- Online features with SDK --- + WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future. + conv_rate : [0.6799587607383728, 0.9761165976524353] + driver_id : [1001, 1002] + + --- Online features with HTTP endpoint --- + conv_rate : [0.67995876 0.9761166 ] + driver_id : [1001 1002] ``` \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet index 83b8c31aa51a5bc273fdce897fbe8a8473179d92..ae8f17e45d3b840f99f0cee550030a913c50b232 100644 GIT binary patch delta 24528 zcmb5#Wmr^S_&55YLqHk~Z~!R@>70Fk0THAdB$O7Ek`U<`6}vkyFt8OxL@_Z@v9Xm< zY!p$kQO^23C$9fF*YoDQkk49sX72^g-Yf2VP0vd4k88w*!S12f8j>2zEHu_x_-ovi z(%`A`cvm!gBY1AT3seUG)#7#TJ{hIT6eWJxq`~A5)p@DSY`PmTYB)1tVa3!DOrB?L zgFdsmzoXxf>2_3kj487-%YD8%Q)HBPYqXe~LdP}@TQRrZ*tM;(4u6I0{9ws#&g^xb zoc&t?v-(`Br5n?&WKF>sW~WW_2`{Fo`v=dL$=_k(9l&ggi(VYeOwd@+8ph<^JTWAS zSv{vaKIXp$Fns3(R`2}b{~>`XI#h0x!sJgrSd_+W((5=gMaT*XPXYZNgqFSbCK&iQo$v+Oo24>ULsbP)G1k>QL*caM=H(#@m~XNbn7>( zcdpLS_{S6lR>z9{W9Q4<*(%9wIx7@=E5iy2MK+d$m^|y`0wrekyOk$Ym~PuH@- zG2gtjnIbi##lx8V>yfRx|20rKcZfc#CveB(4Vk>}Z);4L)dz=sFk`x<`Pz(Tc8(}5 zvSErI?K@-7C9{r`Y8GdSRp~2mV?PVnf=g%S)IQ{$Lqfa%x@?AuzKf9-d=yE zXp?ns5R)I7E`yvw2s;NQ)}u`rs(jvhT}~B zl;ZxA|23e$*W)a!Cp>*J?>v*YYS67q%xZsu##N@9R95VDX6NZmTW>K%g*V^aW%5T4 zSUmX0p8uvdqYECh0}0g$C!R5R(aU+A%xabMUR`2teXU~GzASpp4s^~mXnn^NxrPt< z$mI7`#`iLt_8+bJ%1lUo-Tj@()77&1&8&XlUG$gfwzTLBZ-AZey+=}#DdIixm0|K5 z<(CWPS)nP{^`Rm&VN|A$3X|8hFK->0F>PKD79x=hif_%b~vzhc?> zk<2ECbFwDPgioIX&6vF1`l~IO)d^uwt(k5^D)sD{opjpffIX7&4tG>Pf9eQD)nW@l{EwM?c+{j>5^CjW*|FDjoEnr4S?Dr6?m ztgd1v@5d3-a%Oey%N!BYEnTBw7PFJ@)jx+RdR*wSfXQF6Yu;jJlkcOO%a{ogavH0c zyi?AxYnjyrQ?_nkx>;1c*~IL8)nc)gDcanhze7x<5GAxYvFpE^Xz)bdK6X6e$`Y@G zOx}#Miw-lZoqO-rGu^)Mha6{i?hT1Q$rL5etU1Hv4?ogI;AD-!a|F%*#G7JMAW%|HKr1TrB&Q$=`V<@cVxaO#HO!7po^|@t^iHdAEc0 zcsy-h^}HEr5==Mux&zY8&R;LS$}vTCYA%XQAwR=Yq|6FU2GcI9F%zEEC}=Txs~?08 zWmX5stk?Ok0cocWKC5@0NjDtH6ir{BVa()PT{}FA+4T1J4-00(Rx`3@@?zpD?U>c7 zi?2B{-C9m7yD~dx^+Xa=Bp9*Dou@72@xKLidH&bH0g>rA_Cms>!#RFT-iQ}R1DVwi zRr^DjZp+4aL@+zY70esY6p7W`jAimq+*glhHswjjCNUGt9Ji)2c`wu6q%*5GuCvJe z@A?r}^K)3eQ}K6W9#eFA6t9pcWET*|dKI$+O^%Bel`#`~Pu&$Vd3!!;&tg_5>c-Du zx(yAgna}LJE9_px6fHbty^P8Cd_H~Ue+~4joLR%_2}eC7H!ykG`M#T&)kZs(RWsc> z?mgJf>|86Qvx_MTc1Yg)UqgJkv^@uyP3P8rI>byUxnf_>1 z$ev;K&bWnv=a?dmQ>!jA`8Pj2X=XOf(a~#RCLl2FCX@HG;=mnd^`Y9Y_n2;zJDndf zJM~mVPnn`8?w5t_tiWHHr|^RHbg#txwuioA2NERjt$WMlotEtAW>!zLH~hqOvz(Oi zh1vOf&EaoM(U#^PKbidTKe>Kpld@^$05jog^fhrYcK*x-%F@hgmy?llOt-Jy8x@$H z`-XQ7W{OgT0j6rKz}KnB(PTE=KX`NqGhs=`pW#fNx3UMHSv}x3&w%N6Joly%vvcY; z^-)ZbX={uHlm9|us}-|pgWVfjWwU7vjTtD{JXx)ruY-ufy@N$?uj8x-ko7J;mqp!{@qbbH;;1bSZ3$%1JfrmMMv6C zCo=h&gC$d$O(WfW)0qivQbE zMw^4#i`K&b?={{|)%Y^iUt`1Y|FxW`cj~hperir3v@fh6i(_5X6JO?>2+Q7&B4xw5R5`@}&TBph!fG5yazh-Bi%W2u0y?R7%X}&wC!&XrZb2tJYMITVR_myZrxxh_{MnCjRsRxS-s^JRvqTP zEQmzG@IAVy zh2~*gXcF!^kHWJrzi6G6DT;M`Nh($uG169;=l_+4wkpF&bAa?#7gFlw%fxHnCg|;3 z#Fc3sqe~NdD11~7#=nh%$F6LunK+kn+BC2x{ydF1AY4du#%9A@Z8IIH%|>SJH!eoI zgG=14ftPh9Sm+u7Pj?U8+G>P(@3XP4vZ{UmP&}7p ze%Fy;Wk2n>-b&{O0 zR>9`yhpGLS2NZguY45E0uD{NELV53RPWAd1Zl13*D^PcLzTkCX08Xt4q7Oqa&^EIj zl;!!54w=lMPE8Rl_^^R4G&i}@!{@Z?Z3opk6;S@Mv2ah$MMmg2oJ{m3@C|Y7h6Hu% zkAb$>6K?On9L%^{O}q3wgjnCRnS6cgsdH{7v?lmsu6+>YS^Oh!`zBh|ZjKQSw@KrC zC%Fu_M4UkvXE`+$b@r)<8}LGULmc+R=+GG-Tlij2#*UzG^wH-4>F&4Yg8aqlQ1VCG zz56<6^h-qj>laeDOaShlccur{kIA6Z8H1h)FOYZnU@AJgh|5(n!@)bH7!@^x;tDuA z6JbtPmPzp6IT z!HnJ=E_tS{WcfE4G7GEepHmHqo>h~=yNE~-TiP=ihP&q-dJk8 zn#@n`BC!!mNn*AYlvB$v;(!x|JygIA<1^%^*h^RXs%yn5>3i%k-^&K$70PLgcLXeU6arQ|Y0&e}BnVjL zDjf8Vu1y!waIY@Gn~CLky!<@%ot#ALXPRLOmx`oEw)i$V0@H)MaPV(71uJ#ZmW5T^ zjV=Q$dtiztwIMVoJOy?yLSU7cDOmL49;JQSLLUoWa6Uq10p{G<$u(Duq?Ab=G;Nv# zw_!*=bQh*0ys!k(ZN6A5%TZIoc+TqCBy8?Cz?~n;xZGMwj}1<84@T?K^F$rWHH^d? zA1^GgO5x^J&Y=ZIr($t{6aMCZp&6kL$gShSs_!Q0Z`wv}pX11hn(2(mF%rf{uI1c1 za%rNmI~udkx}NQu2<^#HXh}{&@9687WC(6d!VT?gG%irV1G93hcs?04pg^v*7f2y11bc3!pmJ9` zxBpJG5IL^F*esUIExPK30m&8g>d{BqvPT~MHJd5->uNGKjDnl40~7e`b!JbK$W5eV1NvI+Uy|h94?pFjsjL8U$Hrs#{K}%EKXc zOAQsOqhSy+m!>APapQjo@00TlW%M||;T#IeFn3H423Zy0uxB|WU6Zge$N^__kJDnk z;9QzldmUecNGX$UcIq_;w`5ZFb`qP)|~xhL}7R3qzy!yS95*Hwxh zg=`_u5A1(CEFKq!Bw@=+J=%Qi64h)ngLK9O%y!ViS+^?`YCa7;5i&6RGzyJhmFde0 zeLA|Pgwk!(5Zp5zUiC_x-Jx$}uHHm7{o)vLsgpkHexR@lB?wpOz2aKF1VC~}2u0Yb zP{ie08b3xGJ`1kU*Uv`e{^>8LZ;^|Xz#O_<-%fQuesVQx26(>b1Er21i33HJP%N53 zDm%oeXu3B{@*h*gh_QG(+CwmZ@0ARis9x(Ahy_;k)q+RiEpk-n)lr&&utzHA@4N`K>hORTud> z#^N@);p^=r7`O-*bH)0FG)zYl&KLC2bowb*`)(Jvrf`Y^F(X#c8y{ zRhqte4)@&gIlU((j2|?K_I+@L%bD}8-J>Qxt|Wo@1)Ix zI?3~t33fb)#UB>~e9$&R#baZtYdA{NhuPs|bbn~ji+-WZt`@`gH}uE^c-G7g4~yQ8mN-n+ou<)xqw*0JP8dq2}^4oWp}pTyBLC zonI#0Lks$~(bqMSa?d>DEMq&lRL9A1t31Z(DSV`OgI(N$YBi*torKfPd5BQmNM$!K z(yw(RalJZ0{@KQIz!3 z1A0BGsQ;o#9%k8i)xCgL&XmKA4n52l%fgl%P23L7!O3Q6JX(@LPGJ+Ud(m=A?uo}? zhfr)$6W~JDK6=+u28Gz?oa+R8l0KD)CufFGmGUyywae3uFF%E}^l&5{j2oZ@SG}m_ zupKtFNn+xqE98+j4XxQdG>{mDorachdJzv(9V1Nko(N}IO*pUj;wUbSK8kj$X4$kPKyYs!EV$@41n>eUVoPeEt8EA$rB^SwUlnjaU^u!=64 zVmT63C0=+SbeWEv`#lspk?74f8O(m30onEC0`)UT$z`Sz&AOqE#~-ASej$!>yN6+% z`vDRiv&E_*^5~(7SQeX$H6}?|dwMfBLt6^wQG?Mq{{SsF&c>~!1yol(5!beK(#`Vm z>`8ERbf^qD)8NJQ3!REgnu-ZQct4_=ln9ZLI?8-;h z(1(I6?ZK23R*c|)9|8mMRID1DNVw??{^fDF5Ee=AAFLr~ z$~cwtM3A5RmUMTNay!4gr^A0F@o?}SdUMMWdB>}Tf@#9tRAzUY`+WE>*%t|L!&#n6 z4dv+Lrzk>CDA@X5-q!vhD3vP6#P|08$KJN=9d^3D>O%x?l>}j6;8#+ zJ!n#6JB6mZA)?m+Ei1<3{>+bD{+a7!u=E;zTyu)^8$63HW}YCPygwr9OQBLLi|lRw z@D^@speMR~$c*$v)lS0X(_d-oBol<{Y7*DS$2jG2;H7JGjvFS>9n)NzF@leptRG~w zX&JZnNjfAY*U|il;dpkcio_rL;$)H*bcTkse6k`ciuh3e zTZX;sWU%u`B!p93i|KL3L>v&`K`pn&;C)vVW*d(~_=y}SD(G;pE>+T{k6(yq^N-uE z8-t@q)1fY23MFTM6z6WF`SU-Mm+gJpG-)J))i;vii+N-|#uIA_6w#8Qh7>P(6rTT0 z-HioU#@3V%8qDEr{)a+$c;Rw|wh(f$Vw^X&3)ih!kM^Cme6oS1 zbQRT~9gWBC@uv;^RBM=|%?9QaHvL72T9JWhCH zqKYC+jYV{~>^OPrte{mk?r1wNPaBNGX!@j|R5(hQfS#435a%-mH~Y@gzF#qPW#Eh8 z(f2u2tiPRxKgvUNv>4pN-;&G1WI=nwaM-~OeOCj~mTrVwPp?pZTMCNuzX@{pt)P?B z^2zFIIZnOwg6P~o&T_^mEWTNQrJ5H6hc|npwcs`l?NWr{`5@YL&>Y)^Glrvib_Tah z*$I0OhjPz@Uee|V`LNsagd#LY;L5f%#Jv2=-HmUczLp_aX4Fn54tJ<{s|w!PFQcCE z`qclM@S->YBbwUbUd$jwH4-3HL_-qt&H3 zkls>D%fwr$rzIEfjyKb=+}ZS@&KA}xIp|+~nR9BgfxAMNtKzX>6va-$nk4qimgI9W zza(J7?sCz|+1!me#?&xQLkO!PU$}jKN_9Rf=qFzln!jUF7cdrw)I$*Q!UF44-BA1W ziJd+8%W82Agkr0spy{>7Hjo#YPJ@1!R;wc)`>!X+hj}?UZ!7K$7vql2dU76 zm?e!Ukp<^UsJySC(gA^9^YR#l0mo_jLp;NhtTp3nS@nRz$ae}Uojll zqdf6&i2=9~?`W`Loa@O2IdJ-6MGBdgW{gTDpvObmu{QLKf$| zaI4_QIw4~HY+?H?0O?V-oMiB1=+?H=A*CYc5UCIp^+(}oQ775Bhv2rFE6Me~6bP@+ zzC*g+!6e#njTAp!=gif#p|h(5k4vWF4Vx8vG^gXaMhN!W8d7GTCDy(ULCLTT*ePa^ z*Nho7F8L81Z>vC>*?THok&O8-ho2wWS2{CCt{9inyIj!m#S22d(RxP0f|Hmm}XG2}v z3WCVflzw9?Ei^BI+@5-_cvBXpt`=%?6SFPhxZD}yF2CsHVRQTq-9m3n&XVz^c=$aw z!XpDCWCaxB+&4e;8XLo|L>EKcE|HI;3s@Eq?n7Ray1*R@{y^oJKC+hKa5G{acO@_$ ze;s(3X7z${EkkgGb!>y|U(s9}LtL)WfYE9bj1saFQ$7v|1P?n2t@Sw#k8im5Z`6;aQtH^SC>3i zhy^trNei}T^Wk7e-Nm5;Kkmg9|K>va)Vjhmb z#A3pN=gl-{4j($c#@H4hi=QqJso&k4>Y^R!WD*2F2QqPa<7B9ZHE{aBT(J55H2Sje z1=&1yg7M%(R23-{$J}UR?v{cIu9PewyLK6zzjuZ*9p=(oJ27k&8;s;J#l@#rR6vsezH(c7u@={RVs$CFp<2d;6o4OOmiz(!FxbUXHOWuY@^ zikCBG4M@PrFb&GRLJU|grqZl>Zi(<@GR}o5LaEUbv$h|g8A?JN5|tdBQ@+e3olsA-gx zT!?i(o)j`C8-ruFQRTT>n%O0e=m@NxULyB89(U@Mwjg=jWmyS+YtQv{3n&g0|kTG<#qU?O(HvJEdoUlZsg!Jr1FpM%8q{&;mE1gzk1%Z1VKNx$6(e zVnqCeTaY<=1e}lNa-SpZ;h=P!YK|(QKmRoS8F`m8eDjs`uF9cs zb1q%{2Hb)9l*8xWRiMsL%?6spt@O@5sgtkuS^t{ItjFDk(ww@+?F=)_`bRCiE4r6J|T2 z=%Y0q9XX#~@$_MO-4Ff6DQK8lOfrJ;SQ|H#V%XX)UUMe*Fx-*r2zJ8y@NdpTT6Aa}iV}*kJmMmSto%juR+v!V_~YzRRfozR3v80VN@qR^k8*l$ zj!2logHro7N_rK8n8ilOjoiXH=k4Jd#Lm&mnrKqCQ@~5V=iIFDv1nNCfz;S^L=C!3 zlg3QJg>4m(kC1|$_H*jEV2i6$)VYCSzEoF&PD>k+_@*w$C-7Pi==TS);Tp34XHIpm^HAZUOJYVp!-i}Bk9YsS&;8%0Q2#gS z&zV|sB#Ie_>RU<3HJpZNdUrVQgMk>^Aw~J4$X-V-~DoM5@`HMwh);Y9xiPVvDQe0HxT3xkPp4Fe>gD=0m!gZxDr zIGvD4{KrBFGv=?Sbx~4q|H4D!J2^mC2K{Ppsn|Fdr=#?-;r&{QnP*Ft!Tc{Ox(=kDk(j5$7MK$Ns8gqQZMY7@q?-#j)G3@aT>%P?(lIN zSi;T^LB|UqtMP?1kX}u54WyxXZz6TLlvCx515~oLlESQFsZGA0F06Y-EoaB#0y40U zEh^7U|3$A($)M}cH98w6OMTBOpszI+UbA)!<}DO@;=PY9l+52!)(Sbim)=N8@gW#q z`JLmnFClNm)7-CgN38fOL-{&a1>zYFsNOLaGCgXrl%L0qdOSe#7Ok}I&`?;cv4*&K z88)9)!Mtv5&S70E^~qWYI=1WMo~$LJUrJM~Nf1)+h+*)ntb{v9mtNSZ*Rttn)Eg9 z($EK#rlW)G_L*E#@mTs9`;yY`{SjR3K1atI_mIj_Us~p;ij2fqwDO!$UEzyq{Nbo` znGE3Hm@PhRPZNli zx6wWePulod8eeObadzM@m8~kF`v|4aEq|$Yfe*ZOc<5Ou!Cm?}g+(G*;CSR{glnl{ z@Q!b^H2Nj2KlYC6NoW0PXH^l>FAsximI~5O-6q-j-n6Lum%uwa7?EGTk(x;!O7`Z% zqNV~>1?-=?I|OkHX~p5;2s$56Gp$2$eri4p_;Qf{EKYwNIUHQA29F1tTttu!zV*o> zcIO6iJQ0qV;abp0+AJ7XFoBE@%R(kWD2DbNf-e_<%qTI6i^zj)+7U7{*hq8wTey}y zTU-}qP{kDp_V{Sx;(kM{)GWpw-3ZJ}PoNl?m(*oZPkK9!)9W4|Zph{3bo9~{fsB|9 z4)5}Sp@{&!@As4claV-AFdet1yI}3$D-<%pnyMbwObyg zdoGawRuP?wTTaJW`uXz03v}6C1-Vkw@a}pQ1*?|AMlB42AXQ9okHfyWbF?-)3>(Mq zC&Qpxf#k~hbgd&7ZZ`t(CC?LDAF|=|b-G}aeH(2HxX;2|K8Q@v!^x>@xC{NA+(GAZ ze3Dowq~H;EX~_c{(l#l;hK2vQ^Af`$;HN5jT!^d`4 zTG<^%TOMv8jmD286(xgoQ$7|v`A%KJDA-Iag{6Kl2JKkJ4QZK2$9<;4YurULdDzMs z`A>(E%3i_GMPgX5P(?+%g{H8uvxPPRDdu~ryZ8n*O^xGLT?v5o>HxGXo(fybX=rC|S#FOC`CMEKE~kdnne_Bg7n!WFM4$FAT3i-^t1?jtlgop{ zw2id9z6hi0-jT)|dzj@-rn_UJsLJOPcStz7m3zFuog3FS3dvz+IC->&B!8cvAJ#$C zex{z5uPnpux+pqftBm}#0K|FuVD0B?EAPiFfTs=2N$X%MQ#DN zVcu0PBhwqs=Oi%)e0+bW1E&-z6g!NCg6%J|yJ?DP_j1rU=sx8)w9+1%GqiE+K5osO zI4J98BJol=R^2he{egXyb*_qL?_(46i%%(#E$%fY=|VJeFXhc0jnccZa9Ga6nqQ7W znqgc^>NoT86$yCaT86dnC(*JuyXg5*1lqe<_Ps8>LaD zkwj@veUUM36lBgDL*Fogq`!Tk`-QRG!^huf$Hr4!Ab&gUuvJ6!iUm~K_m(tAq#*b} zI{pT*T?(bUC}^FqfxZ~};>vwHD(a9!kby3a&1j@K3pJ5j=#4l#Ns#+%y66!H_r0NX zvGxxgEcZmmf;_tX<`BKj)<&jk1!h*=CacRl#F?{2;BJ;RJXuFlBd^oKgPORrY%3kw z9)-l7T5=0EM+myefbW9J^~bqJp{xT7R|zqBOFjY%J>V6;oy1>z(YfKjX;Q`>%Gz;? zM87|C!{XxcPhtYk`Rd)5NCH<**iiL8H`$li!`82uo+zXVtQ$gz#pdIO>vvil{Fy6wJdgez=OJZm9t7rIAn9fbtT7VOh>s2Q)#D7g zPOreJ2@gp*Jqg~w)hT!`55q(v3cXfCj+1H2Tr-rh4CB~t zAN*)v;M%sql&!m`K|L!RttS6iCl<^}mtLWN+Y~S(HWG~y#w7RF95=&@5Lsq}`BgIb zyFi$Wk?&=(yW0(oJ`tqu&N_m4dkWbTf-_^yp>?Jl{_pHz|M(A)=~C{T#(u6Z<~z67 zy^p3W*+KInlQHO_5^U{5@cij#YRW!CaUCX{pWQ-hy}sV z=xfe)T9)&e)|?i{T4B^UyvsU9;VeL7{W^vYwJ1{37de&>*Fw`QC-}`*eV}1N^(wOJ}$46a-BhOB1|Tx<1_I3zc0K@W>g2U8)a-oWG44rob9^`h&2$hlLaL z1lTg#3HLmF@K$m#yjXvJefeWL+_{iW^_J3mD_8vfbC^Cxnm~004+q!TVE^KY7|Zc6 zw(cQUns$|}yItWtToWfICBpN_Qm&~i5K-ZmsBMb^XOjDZB2~Px@lP@5ZYNa0^g9{2 zSEYuAlw!Qf4B_r~Nm8o5Ev0QJ#nCafoc*9ZG}m7j@6+^XWlI1H)NP}V5vRzfJ_VyU zvGiv$4^^vlAWRJw#H41^yN(Z3sA569dzUCsb1!#m&K_>7jUyIo7edEI2M0^4sAP8= zNmPYFLFjghbPg?{ldCe(v@V!blC^1E^E4>Dcu$#E-f+8QuaU_vS1O#INpILrvWiI@ z&JJHoewV$8P7S8i#o;&-GYPK?GqCBc5T{b-)8F1odMoEnxeaq@xj=$m80Mnm=2To= z<&5cGZ1ae=I$c>5L;I%cK^Q%hWe)7whMk66RD5eIJ!4(#(}hJymfOv>i{GXUhwpTv zt`sYRHq!C-IA}ezg5~HC*k*ZPTIer<)5dX7T$Yd4`>u3qt{)bZ2P5vl913fwrM`M@9+b(;@UKDeH=E(?4n=ELe6p@32`)O2?grkCyS0ru-lpl$4|Gow#`o1 z=KY2u&R?PuUnwk4kc4)A1&qoDqv>Qk9*As^EEr(ws9 z5J(BkuueCK{Li@|&?tiBXMLggmu{T!rYem%TZ;75X{eJMBrr7)hEVjUNGS9^r+p1$ zS#XZ6Xnu2$8RQAQ&h3=DCK=BgY&c$P8f2DT=h9B4BX6N8qE@&Gl2?Txq%)oR_69@q z=N4|CoB$ff1W4>o=G+ZhsUlzqMx}@06x;YQ!c!Z@^XtjXQA7oAG_b#`ntDGCMV@f2 z80vBfvhqsEP_x3%4G+oka}g%$UE$(&O0b>n6f+(^0+PY6xd&%+A-=N$?|)lyk$tAz z>w)duKH2fu=K7nY?ufXEt1579*Q8~78cq-F5x-V|1*F@P>DRR-3Dm^_xr7l(2>lmn-1o{@DUkku}-|JX$J`cBPtabH^^_q?|8Y2KWXNUQjDhN^I6Sh^ zM^0rrvbpW#_EH;Bh90ONvWyHreuv$<6NU-ddR`Lw zQzB_E8&(a!&!D|qRB^}cnn3D-KRs3Ms>1%3g(80pYVi_axNV}2Y;+5U*O&qm1qQvpT&iI6>fTu^RUh(-SADQ)U` zdOR_lYaP=k<+I_2-g0lL!(b{ay@V8R#G4owJ(sDt}HAzxI!v= zB!05_|G!Qav=i6Goq;Y&SJc3_Gj^~LKTQ(`ZRf0ieda7%@6n9NajcuIrX!s%NLOPu z^)@AA^Za)tu}2>t*aEi0&>62k-JtsRQnp1}_=D!_CsWXGM<^eg#C3d-63E|;7WkOT zp?$4A8mtE>>H2cPtaJ;~%CkWE;2`38S|ai(o0uMfK<#xR`ZEjBRg^{>*cL{^?vuni zOFCmyzI!k>I#$l&}9@eWz;#rdwCbaNKV%HUV8o-g+ zcn|!2P{9?jjXmj0Y!EwF8(I;6NYFBw*o>2|PK`pQZU{P>4Del7fU-}{_~W;bgf36& zsaw(=El>B5!kpzK-eV5gr>E%L0uuzylOPAP#iZfJ(Rnvd431VsQi><^!=KUG4Yz4p ziV)w}+OH&x5Bm@GH2HS{seijlKUv)Usq`mu{IrZpEw56H=p+?KO~S8(w`sz5X)IG( zOW*IE;XWv`=fCHs5<(A8ftu`9GQaDNm%%zPQ@0^4Gj&W#6VlI=9Ly663 zsxvQA)W#(=Xmk&$?9zn0?{$ja?}DzOS@@D6pyJt4s7;T8%EWM}v~Q;s`-kCc@>(v$ zpd8w~C1m$y2vp?4;dhyZM?VV*XZ&W+zi(&ATQUU|R#Ge;dWSk@iIayY7uu^tY}c>_ zKD@d|G^z--8|*Q=_AV{ZGR5OxX)rrA4N(bh_#%--+iT=#Zp{G-5Zxqew#`*1Rvj}! z6Dec011_+e^K||lns)lSOGVyV3S93&rc<*iS{U?!BtN>*=h1uU)}gs{DI^V2bF9$i zx}U1oN#lK#0~Jov;Dj|c__LBF@?QI)VdP~hyWdPzAFbgwX#w@u1R-zoAUcveks>5) z=-PcPE_IeO^hN|x;*up4RuGAk(-o;LLYG>`Xdw8QBb^Up8v-p{NjNQphs&Nu*dM5b zMu|x*vZ0R&XA|&X%yk;FvWZ69c#)lP5~Uv3g;-P}N=FVujj1~B#g6p zf?{E~$PE{DMxt)2G4_tOL;ig?EHKtZ&bS-&wz3%B>$I^&MGL;NMzne4N%9*TgQhdN zoS9uK?c7|9GC5&7MkQ)-i7Q!3$}kChHb!rHV?Zj~J#b#`5uFvV1b*rvx^f6Gbw~qs z%AxK1TB?-ZMc1x2lS!sBc0b}_=+=?AS&@!lM<*JwEQt({@1$Mq3m%Gr2gvM>A$HF4 z$GvSMAbMIsK9^lEZeBJl=I6T#g-@PvDb^=wIa{SzU)WCb0t^tOVa8&8)-X+GsVCD+ zoIIyWFCU%fKD5+RLG)AvcB`ZFeH7V5NkUSBE!WRV;q5>MUqTAsWL}ZDk39tk zdZ1yoB*jaI~``p~}2{^w^ z*vRdFI|3162PylIDO@ZkQ}$0+Xzntnj7>gh<^Li52UC$&lz`9)r>ScBCQ6@W4K4jeOF|yYlT@}-*xWo?_5CKd zxyqY6amACHapETh7#JXX^$c2g;wU}P%VbmW3pDA5lOS1x&2Xey4=3Y>IH5muoYZM~ z!fb)p`3HgmDL^P4fW*XmbpB8tRsYF^FedFTJ)JX--uH*o)vp`4m-p{;Ue?J7=bxY@ zLoSl1?FOz!wg{ugm!No%IULSwQ>f+`*v2^uDt3Be;Z0-idE^-AcUaTeSu*&|w#gWM zOGZ+;9Guh?aBOG@R^HLV_Cw=H8FKhj`Gp$)R+Gq~5Qo0|3$bgRCuZ&Jp`I_&R1l{G zr?|;bxYA5T%)&cMLY1GLRN1@h|37<9@HE_=1O(hX@a7!*jk`_FLaKSn`MXAw;q5dNc) zibn=NuTburt1mNXGhpKqfSU3uz%ewrGj%%C@P z6ue&OVMEgv(mrB9pN0p(_v$_h_Dn^Kq$BoqMAO@SkEmVa18EqufvR5v`+6M6+t3{E z=NaPLcpq9Q+}+Q8x>-!tRiYSG_1A>VppVN+!#G9bIF1*b3-fbn%~M@FS<{uylVpO0m8lA{UpsM3QC9kDaI-f zT1nP8HQ<0GBU=;&?4>9pjwU_rrmx$@Ddiv^seivwOG+)Z7rJxq_pMyN*^a?q`61Np zr-5mfd|DP4L!XzvrX$e6$Snd4&f-v~*+cHy>OyR8NrB!3O{gA>0OC4m!Q8tp14pP_-O-6juS0D_2M_)PzzB2Z$dMj-2-a*tg^VsO3C> znoid>9y*~(384r`GlC#p6eI}&=|!5-q!$5^r3;7#P^3zWAVskNf~Y9XUZ@MAAZ0}a z)`Fmj;v%}Py6om2E%`abZXi*%FdvwSX8E=v;KcrJ{tx~qX!dd_sq~zq`03ZoHd>f)rBQm>9=fCJEY&7Z zb|rmciI*nvah|rDB0=Y|d_bFDbC7oEuqypQw-c>Dft~i`XE`b<+lYScJU@N&us(i& z4&P}0-OHcLKbMi@T>ctk`6bNX)9){Tid@X#_~*-?Qj)xW0RgN=12JaD6Lw0&TQ21=?|qsXYfS?PkLj_8=1830Q)AxcqbgohqOK zePj;oGlyZmg-M1sVIT_5-~b2gP>MmD?C3m0PMkS|MO5>yk1xZ!`mtP}YJ zC_>kQ{1m8wI?VT?=Ld0M9Nid{fSwCvVP6J1gGz;iNg%|axQ%cKWgC{=;2Y6(gTv_d zqw9l8B7cE?hdCAg7N`eXmd}A!>=j~G1&ol7Ag94AfoAM|4_wg;p&x?gp_j(}69hma zkip;{xCNgLZh{;f(#7((;4bnvm{+6w8U82uO7v>TKSJLDIj|mmA+!SYVfGj~8!TUg z?1D}eXWOCCm@&oi$1@b`mv0>92_OrCkPm>f@WIgYIP3tmfqoDD1v-thd*DgfA%Y%s z_QWBm5zy)~#yaX$dHNXkXXQ4Hq z1+Z{q*3cRBQSdo{33j1lM@NR=kMk2y6LeDOdZ9GvC#VQuW5dV)IF>@deiR;94h83t zN0A-j<=_?ILvZj6JP}%f-T>WaL@TbRBk<&^>~3A+Lqr z_{SqgC<5*%h+qW{I3ZgBFZhEvM1=1F$G|*xa?zy&Rm^VySM0x#On1aLTx1&pg zF99at3o;#Q2+e{rx8aYIfQ98Hlt%E6pk`nl(8irS!K?=TWoRHeDRgO|7u`*;8r^Z^ z`|yjv8TmQ#HSFJjj-m4b=RiF2T3`iaIq>{PV>tBSf(C*i_+<1Kq0hlybiv5Gar?H=%gC>wCdeB>1^9&fCm3T8fx%f6AE0K) zdhk)Wk#*n*IDoDid>xFh36}fd^QN-A(9S;D3ZKhcci% z=r-U?3w#EA9~2|P19Z&aQ2YvGgklW70F!Uv`=NEvOz1dZfB`TGe#ZVe>^8x(LnE;# z27eP+WA*?Y0~!JpFkX#m@S9ry*ECH6+4rpTG_D!9E< z&qtwA<{ ze+^xOJJ|pfKow>O(4T@|3B3;WK(`s*7d{VKzx?@M5#=-r4HUP*8t?))RswYc!Hik)<*Kk@`r3-?dZL|F{VP;x_Kq319(!{I8Z zJv0rP3f+hqJ5a&i5U_x^1(TSaM#i&AIE~#fXft|c_!P_@feSzoeLtAQYzpjJ{`yZD z0|{UUVwRVHJC-)#uojl2kzL^hz$N5n%vM9|kY&Ij3upe%H1&?n#yI#1-^ppD4E&|2g~+_?mD3it>d&;SR)MkA>XPkgR+RGa4y!el$?VoDr@FNIc&ZQk_o>!Z zpXytCu4SrE?P8bWUGeJ+g$!m& zW9)fnmkRO>qmMQTZ3=C9KCo$_OKHX0_k+&))W?(ix6r=4>BLXDK2`P530%q!22KK- z9T?nOBONyJuC5+55m-Mhth$0hD%>n)XX0o^_B!BbE^bscWG=%r8!WU~l}Z}sR9vqx zXsKKrIBca-Q$1|0PF&>I3~reow#jcJv21mRT%1+(y90;qHqKO!*c%-i8QJ*u(+4pJ zhMD4Fsvc37n0P-qfnSa#sX(TLs7MH_nH4U4Eu- z!l5>OdLp7?VRVA~+UJiGjCq2@(a25_9&w@@ujA3E9)-(hTVgGXoTK~t31;itxmsjm z`ct3E5bmcEyke7>De=6zI+hmqV$V6c9*cYEcf`x7u%pow=PUzsa0aq+0D5a(=7t+)DXt`6@#Z*RV|QCv_JJD&W0$n`kI zacuXheNi(#-G*QKn|*dJ{n)cz;MZSxd{fxpc6O$4k}dO62iiU|@1{=O(C1Afu2mrk z{jjdrcfSyg*YAL+{g|U5X{v^iPA)O&Oqcw~a5^X*A<0Q$BkIXyNXO2e^#398qe7;F zoV>rWlC_DV$?Eb1#q3II;i+sjTJO}Rb!5G?9L)~Ns@yu+5XKfWNnIsr^C7pYLs^q` z>^V9PY0qQJLyx%RF%94L5`&CHhpO`FVT-Z_ueTRD@)5XIr^%+Gp92bI)%SPn*R{l3 z=b1Mhk+)rA%$;X!G2rB0_(cAyWf7OknS3jo$0kCQ5Z~wQtlXs!43<4#RUeCNEEN>S zQdXKd?4scwfOX1ecS}xSK>soL6P97*QRg5F;XLOw_-zrVKblmNPt#io`$6JG#m0 zWT!_=(NpPTaeeNmjxl$OGRjR7t7jkHQczw%y7SOE4Y`NaGT~Zvg`dI$D1-PZnf}wmcaFXLPw)Sc0E-%$Gy;$ zbk?Vsn=h-T@wk@utH#H~Zjq+2aaOoc`T@S&CNfcYTxCk&>YJoj$vx_8&aHpw*1V@Q zaW+CV_occ!)3|Qj`Tob+yETO-D|e;u6SOH+&y!j+FF4|4rK>a5>-pB4zWgNZ?892^B#~28vp}}8bAb9v0m$E)MLRitM;Sc)9As zXvO)fgY_FSZvIA$yr3LARUsPmdG5NV)Y8WZQ7N<2e0N++)E}+0w^@0J&tj{DJTdKd z+vnOKhOo!os;C{m@JF_3N7~#EWOq&P^ZAMPIzT$?(y(`h>x=1c&x;8#Z6n(c4m!P% z?;vk-w5XTsJJUWhnK{?4xp*Z)rlmoFkodqMJF>%@`gFvW@9bxP_d&1BwvGs+&d9J8 zdg>+Bd$?y8l!T20UR8*lSY+>tk*ZhQ+OoA*BW}g>9c*jlhDh%mo>)z)SvH*M^(#@f zvwasUvZ^DrTkG+A?wol8{lm;CPj_2udkNPZ?Mn^U2IWc;I?8JHSqClkYrI^X5={~B zYO21lF9h$6R&wgSu~emCxN>h2hz1q?D_Or|0j~b=7)*BXOU33rYzS5tx zr^ZL4u}jgAXs4B#r_nL{Zc{+AOo!d`)NrrfCk+ZsY}s07#VSfHTQ%lmTjKSODmBTO zd}>We^4U9MbH*cgRBUo`sEgFxdftXttojm}1BFcjVP|ec>&YsYn(Cd*ra10D({!$z zmwM$XJArcQDSiaZ{`Cr&%6Lh^dROu-SIVdB9F*xy4p#my8P=0{uK)FZf5PG%jg*?Kki@|QtM|evtex|9gwC#`sn6l@_5=hK;RKhCn&} z91rX3n;l9?@~{GSi-i_UaZsK<Qiax9@lOr^n8T`-}~WiT&eaXFb`i%%AoG z4?y7S$3-YXmb-d}gYsdTrJX=t@s~G0WNk}O<_gahkuhSWCLH{Kd6!1Yzk4G_sXsjy z%3m&LGXHNbXQKS=ivo=Q>P3P7>|fVk{t|z9>RG93;~x+RnmY8R@PXd7l18tlm0mT{HMv{^W_l8sHCbs@~ delta 25529 zcmZ^~d0b6z_`lmcQ&EZLd7d=aUiZ5wjhZByOGT23k`ir0<_sw%l6g!*k)a5gnvgj~ zD8mOy$W)xi`JLDKopWB_^VendzSdTI?X}ju?)%vvi-r0Mh4=xkK~{<)ikr<8JM?`O z+eH;cMhOVqRI(Jf`Mma7lc<0|a{9*Sf)X-v;tAahHQSwEFibZe^-@qmV(9W)Q3;vd zQ!l*|l#_9Lx3z~wZx<)`GUUrme9d6Ff9xA3-nZ{(Vpj9@w@g%x-uaG24Fl=_3GyXm z;#YaUXPXk^v_3HNSmk$S2Dr9;WaiiV`}pP_q`;b$i1>HKA)VeNwf zCboDV`@*8MXUo4bD9jK2#&A~Da*!dSRO~zR1alcm^Z zU7D^mGh;-4GgD;ST^VK`HaR?kMZTAEM>6yWO_gORdTk=d;IK$Yp81_4o-lvuo~lvI z*R@?WnngEnMkp{OM_4N|$n{GqF|lSvHxs9isU5?_p`#m=S+tw@@nad?+MUKRyiJf- zVP^i%H_Wu$a8Z?+_cgbwu_&u%jyi*?=L8LghNsG!OpH(c#6$_<>sm}aR$Qjdq5$J` z9fq&nD44#~Y1;d9sqL$3x9N)=&n;)mFnE!bF+VLz}tPx|) zpjmBa!_YWsge?;jJ6x6Hf>h*s~~j>ny$l;}1h}WGJgsaAKxQz~H`^iE{W04??#XcfbSKXsn;g#Q5n&AuN*kkTi|Yc#Q8logrYfaws#u9_(Z0Hm7T0%;dH02xn1u?EDCZ zylqJrDL7#5v{zV!^5AM>059!o0WbgVW#Y~42mig3$>vLzGRR$-vW%f-s_}A$>F)$qFn?(ABj)dxJD$gU zxBYo5S@hPPAHIq)zu9UvgJrbD8fM-fc*e}ERj1c7Q*~VaIuQDqW9I&xy`?Pjl+E7G@L{k24u;Kk`ejVCX&PeU zaF)`-SJ0^-u zX{co4;n&5-S>(HDZWTlS2)7dqMSI4bWTJ!Zzhl~Y^TsJAE{!O!W|3}xMhyf1<_e$F zjLBoP&oEQ2;s-Np=yokLr?($C%c7x##pf7y{|q|M;I_f&0>fKPfjZ{r*F0ptrDtV5 z^Y1@hagjw?sbQBGRE4b?7#fPjFEcUT_}?*=xL9+AiN^xhUu981Z*1egPtDgXhieSm zWMr>1lefE9n_xk6P?UoGk$d!i%hRa zK4Z8OYV({SDi$r({ul}8Xhi^Oguub2%QN7IUAN*HR z6s^1Tnwbvj#cx>Dc{2Ge!&3K&?-+ESjQx*^H`5BkD*ztQk3Gqa}uD=AbzJov`U z2L7=@7R8TV@tr~9Q0NbaV@{So83NkGelh=R?7xy?+qdf9%;&9LH^icD)!1Q%yc70+ z7!2J;{$*mzBUVwErp>t^(87ubg}()YEIYd;QRv^_AE6InwkdF4QG}`cC;uxcJYQT9 zWv}@mqg0GVnf*|4U;3`oAPev6<$;@ zEzDL>m+JHRzyHTQ3d*8&-um5MEVSbPe_!Kmn&Lp3ui|#5vtqTz;j6XkxAW(yCDj`R z{tsyFf@j?8+C{W#rw@D-S8?lY50J{TsknJ+4;}l@ljhd<(J`5C+|aE6WL~w#vh|u6 z6qy3mReOo@3~=Xe2wt_-)5mAcT;Y%u_}`Bw(~Sl%gn!6|O3_5xRAh&{A_g>K{|_>e z6QX=^6;8tEJhyn&Cpx7pg(@i>q%WR}(7teNFFnNNH;0kXt00=-sR}8{S!hqZKo7+q za3@ZIdr=^a=K;A11J{frp<3sS|8{0l$LPbGHeH8Lr7t0FlsO$1i;%Ba3NiUkrarEL`p#M5G-r2`p%DzIXX&cGVR~cqmN(ebq zLsjJoi1CTxtTXP?m6o~Sk5-44j2caSJ`GxF@mR2g!>lX$BN89@9sKLu9*T5BrJ25F;N#_qPlZ9th!#*GwFYGDFIz z0C0}hNc2xa;J%%-?}`UW&(J`qc`Y4q6sAkf_h^pIDz2hh2niPiaN@EIhHjbB^4*u{ zV1*<~ANu3r_FfwMr<@S}q8GK!KIiJ6a{60h|d8ChrIMsqTh5642HEdwInnIyuW@R>`< zn1|_xt#rHH79-!#-ZLmpUF&4MPq2KB&m+mPK z^$o!=+;oYSJ(Na(=o7j!c@(~U(!!tQtyGbiNJl!4P~(N4+}%Chyw$=}QA)nhv9Lqr zgLTyU{3G$#e6+xt#&CN7Upc)97v;>)AEi3CQK+i2B28}{r2YIvT}Rhah~qgfs4f5$F{u+bv)>&xI=_kEf(_73sp^pQ`AAm)^~liWdJ zlvdrP(rrbgR+@_wax=Iwo_c(Y@N%T*#_E_CB8fZqqai$HF7i8MFfz-Dd@VG&-*X>P zyZ03AT&hi$)`1kXOaZ^s7s7Bz7I~YNaf@#LCYPgmO9pIR}G6I=k&Xsd#MVigotUyP^?f z_CpNsj_4rHqlS_+JZOjju@g^uFMPdF|IG&Smm0Yq&%N~fp*gJh$!^eDFO0Sg3!LZA zwIHdRvM4&Sohpqdpl7cLD$UjDVW=;<7CfV^F_X#q+FY9b&=XEaPtjX*O(e~kibGHC zlepn*h*p)7@w^3`LG5}{@|cBL<#$Ny{Ag%CuOz2NJCvT9i93qZ@a_FuZu;#AtQ3$3 z-_}8%3=V|C?u$E2vO6iP$`-=?R-_R;0q*{pNGf!r&#IjioA8(h{#%IrgiDl^8b-rS z3b^)*uxy723bMMnl%_6vz0n%m?M3D#M=vYLb>?hrigOqT?30~IX2pa#1TxPwdU-SG3!jkl% zs*h&Rut4fS9?3X8q*rlLwE8jsGu5;LPsHcY#2;Do$a#n!_Y!g*Y9qO)iCZRTOvfi2 zrZ+1WVfOC3RA`rt*Ha>J|6L#&ZzZhPX`0*&(Wn>4mfjd9=6OefnKT6k+$FcMxW+uKJ(68$g>`9A!qnESq*-ft%ajZ?$oB!3*Q^W#1rcE|69IhgU~Dm6*}AnWm7 zWP4eW+pcH>*R_{Px-tRorE2&knnJDLJ+U@mfVKo^QQD$l@D9kMc>e@E7EwW+)DxPv zR}8&>m*DaFZd$An0q(>s+zCEHf^mtc=9`#f@{5JEFd-feXKs?xy*1=~YC6vM{72Dr ziZ1_IfGD{!^k`imu2ef<{u^0L(Em+W97mxu?hyHO8N*D)7@Z#?sB>pCWoImauD&b| z%=1IIU^SO`Z-A~!W#Yo;h0t51g(GM6XkS<`2FV4%ZT8^XFWOI2kGPU=s1Ynhjd1L8 z8EtN{z|d`0c!*r)YAZZ>51V?pU+#RGT9X8C$=}>ZiC^@s>m%1`#Ygm~KirP^R7$xK zgpY3j(MRQ2v|GlI)p{{BR2(N&gduCc6r9BEad3qrYE2`t^{O5dO2;JzZjX- zQm7frkB3z7FwL*l#=!hk?3r%B^{tPB(r+aQPvX;#w^`V(*h}qE{^XN-kz#j5!^}$( zA)l>K`+gTqVQ0u`)t$VdX;Bn^po|N4Gb5f(I$}@BB4BDajW9e+RmvwR%Fv7E8}v|i zq6z%%Hqkt}{TxlN;q>{vYsvD$Y|5IKLAEzcP(3aK+s~>&eWC)coHysVba5n2zr)49 zoCI!b7nimCBgxm^BMrGQoYyWRx9Ax-R+|cCUI4~d93yq|fM37`+EQ+elHbmlHl=~$ z^V6}(vX?5fq}elr0NhWiLcNlm;y+#!|M^~^PPc*Od_Td@82I zE6|VZFq-qJf$~}+QKJ+OBzAGKmb&n;FD2b^O|*IG5H%=Dk?6$;v`tKcm_j<1n~z0q zkS)HB*g}qfb2Ek3@=xIwh$3b*uH43hK&;Aka_ z)ea#P+qs_F?t7qg?Jml;KR`#9kLE13b7+A@4SDp}QhR&}$^X%W_^EhYRd_?HuB+(9 z;A71Gjw0;Ey}kR1u8m@?7}m?IXv1x9HN^Op?4d$Z3T|VbV6B3P2zOErHIVEDtDIus{Tt*;^Cf5yO6B?@!hyh!PwKi1?s3l&Oq zaC%9Tayiu*!-R=8tILj5*z9B6AgUQs36CnU!*{L#302Qc@VDy*WqxkpEVikVO6C!YEgQ@E z-(5v!muko*HwaQ&l~IM+R9|qDh7qPM?%8L%@lxDaU#MEMv(m; zAM72KjB$lZc-%Y@yl;QGBUe(<;*kv1D>W3K9u8IWEP7?;4yljE@URG_xBM)2XR=77 zMw>)PrM2>e&&bg#MQkQYV>-5mHpRWPcJ4)DNvY)a#CLrV01e~{<1MV6anN$Fk_;*prwQ-Q)`y;}22?Did)9v>O$W@ZSpTELzn(2tCHY2#OdsSSWkvM%( zP~g2emjuB(R(L0vL6=_4qZ#vuXnMOg8pfz&gsuaI+RxD4bF-ny*C9+3)Z#|Vh$G%q z5EuE&>6XoVPWgu`mKUXY}XS|5O=Y=VjB|fQwXI|CEB# z?NDO40AKrCDYQ17+MjEp?co8g(Pb-l*=07_T9;9Xw;0{b3PPk(0%8-}>1~!FuJ8uP zWBW#Odf>t(SFGes7jQ?#3_Y?LA;xa;j4Vg-XVr7dt9+dx{CV8 zo~6`+)%3@*o92HTLx23!@O#!a`ei+r=1+CSbWul;P!wy^&8L<~5Ac08vT;sngIz|=cqEyx_Mm%8GH}@OkZ#Xf z%*TqEzPNjS1=UUTgwiK(T$?frbsZ<@!Rtl1t)xriL$h(sUJ~H-J6Dd;hrpQ%&hqXI z==B@W%blO-KJNqfbj~9ZpX-5X2TfqTK?}yybGT;diEBoi&B4~27NjBl$Sa1fsRiTro!qI6g%QBi953nNX;Z# zQV|NL16R44FfB6B?WW^)DM&7=Bj3|gXqdlG4i76eaiw!2)SgPBLv)9sIW;TX~-C0a(Er2pwk8 z+?ldmte8|nMV4c^ig(K3TN=P9w}MVfKc?8g6IA3X56PpONO|s2YR#U7nCE-AH_fu# z-Q}Q*v4a#M?t~i-B6#R?g+{HOgfD}M_?t79TL(de6c{0uJ#Pdg+LO$WeY8DakWY=_ zLAd$Ak2Ab^z384RR-5ou!)>N;#d_Gjud( zHoi|Y#hOX(ctb|;E*%4t@t)kd;3Db{EF!N94iG7s4u8qDH1hEmZsjU_)CI`V)v1fv ze%)kRbAWt8`7+KEa(_{o?{+$Rynw=M_=HX8=o702i?jA~ZsW}0e)#%O zw|{&9^he0Tt;-H3qk`eOYZK|KOh)bCTs+gU27jaT-2dJ?1<0h z8e4fZFyx7ivMJc~TLz8n7n-onoksoX%B=h&%K~uCz3G!wjLUXFH=U-XZrml9+m}4 z*tZ}H8w*}?*IDPxJ0%**DIe%d&rKTjNDprIxn!hZPs3F?$=(&d-g6rMA;2Z88=+&1AnwH4@T9|xq37_F^qoTKeT_O63~19(^A8G$ z?VvDcH>Ayor|1+R1f)G>ZIu)nQptjy*DL4MZE>8!Rxr@lB&@-_x(t1;XHLex5d*Q~#bCk%=?fe<@(ARW@eif(E#;ap6GhPq+ zJLY4er9SI+f8*}beeQVvM$-PWfeuCNpBlBegBQ+(UmA+#7L<`(%|xe8&=M&lCU^XPTlt4bQKzGw3K^o8%@;gVebe z-t40B>{6+KtXJU(+&zZMcxJfx9LW9}1m0W=jB=O3k2&U~G`^fTfA(~&Y6hh_KCDwx zPqIyZ+}s_$=)CWWcj1AsEnfl$EepH~^MT+%AxYlup^kNdRC>S}35RvDv&Mt64CkYR z|86n99$f-)1vUKEJw?5vH863r55jAtFl(GLmX0###4_1&4Yfvjco;I>M7S5iw#c)S z;6&fKL2&PKI;YS=U7E?fxdZt-R8^GtU(?i>oN@fNi-P*+C#x#|Z}0!Tm(%||Jnc?X{ErPjxBaid=Lv_e z7OJ;P#$yK;gp+I2u%sjq4}^u`X=I5P^#jl*W%* zL;FX^!bm_FDdvszLf(KIc}*K<&a_e2?r&M zymfD(4;OVYc={AANYt5uJ(j!adf!^= zkG(;rP7`VKd12ze%L#>RLnz)Y`AT!DWHIh#6W!>%#`DeJM-yirq-Zf~47VksyW1U2 z57dz8`H7TYzvJl55}MSWh%eF-l=V^qk=IfmqokZt)uE} z9Tqbo=35l|uz?XsQJ(Dt0N>@#tx zOc!M;F7$qd7iOF2@{;*}G^<}9vLWv{*@G)+uF^KzwV!{Q^S-hWH~Xy-`tdbo@BYo{ zR*uB3Km#tiR|8w$I8wC708JyW%8G7>cc|tseq}-;)X))Z} z+;n{QGorO8TgiddC9*fhqWtoF%;4tGC#Vv?tzPeBID1sHTGjV3gBO1YPSx4>S zUU)Ho9{k_iQ9)5I8b&+eKQD8-UNZ&;s>#UHs--$p7pQJn$$RPF&fB}emjo*XDW4uX zj!~DQcbmtPa-ImU{CX`H>t=+k*JEHip^^%;Gx&wsww3VL zP7eoHuW;sHyLo`3s`xJ4$K(Bq zp=+{Zp^)W`KhH*E_Oc`#)b@tlq7YQb=t0)Gk#=@FBkMeyNr)V!z{+?!r>2E8e=kUT z28CzD@lh*G=h<@%v4f4uZ{F5(*kMTE2On44*;^QFtID{N_Y2o&{<;_F9FFcinz%aq33ue=A=<3AnAYtV zdys*Ka(`^gFvjH7j(ED#mw3h#IHUM?Tv+8MS~dR=IVg?AG(8!-tv^M%dZVB3Zq9iCkK9}2Q6rliOj(CL_m$ZhIyQ<#9Qv)N$Wd<$o`SeHIM|INh-t>w;GFC_l* zPtG`)VT6oNt(0?PDuOd(kY6rA~)NIF3$z#d}=jhh2S#Nbu)cPV)AAXs-N9 z{f5=F&P#?y4v67~R8K&%CM(PjZaJ)H~6msVAf{$c#U&8;= zfVmXU*84j3`%R)1-g>+bm4C_1$^iEw4Z)vR#Fa@O=P5WPLrkujd%kK5xBjjhjWwPI zQ%fEAmu#l~ZC#`_?i%gdqrh`^eMxNrIvC-Y!t*xl;5P9$$f9j@F~#^T!DoIT46JhS zxce@39@#^e#Us4C8t|4vKJv4WHCB09`f}iO;@Xy_UO(MhY z=qfqNYfHYs365gr_J;tJ^u3@-LJ#PYvnCdsyW;VNW-jZ{En1syiYcMuB=OXUwy^d- zH@bkb&mZQ3ABG{^-3GDW0x*BEG<1DisnkdcnucRBv1b|moFoVX`y145=z|#D-`w#J zIcWYmg^zyOP*kmIqPry{v2m&lUR)=nwyC50<6=mLpP_|G+B7aj2O9mKs6r|ny=IwM z%vOp-T8yK=(=Jd)Qop_BT$W{V<-@YH_r*-Id@=*t zrNjW{p6xBrMx}o8n?SX}pRRoGn?URD6l9eqv3rxxuuQGvHKPx{2Rt z2|@F<+`UP%2-+n{GOc3~yTKBV>{#vcIfl28KT`pt*er3){os>f$(+EOV9Z1$m`TT@{CeLPw_ztc18sbr!yh0IiRNY^O>N1FAJ znis;Aj25Bt-pwSpz`(hnK_B0}M?*?X65NJ8)bM5jXFB|h+vL8F_)*f%_>w09tzH3& zlzc@Umrba2dpC_U$;7QL9ZZQ5pkZZsyt8PawzK&hJ+s1>!d*^UwewLEzLZpVe5arn z0uXfxhx78Ulu|MU8{XOBa+)h{#7dFSqTRH3>prrw3gc-{yhoY(HZXRqq>&D?xUI98 zPa9vo;J$8hr4}DsNTw$!L*+`$Wi|picFeQ2%=7wm% zMp+G8n|f&ETR)t9XHFVNuF;M+lM%B%9gl@Q=$(NGXxCi)cftf;?z0Yjs1dn@1AD|~_y2q8SRmz0gMlmP_S{iih)z7lvrDkF5?hdvXo|es6M1H; zaZZ~RM$+P0GwJZ5>ELp6QK)r@Yc5r$Cnkw_G*caQtAnX&fdd}rjYmp!2k*URD`(Ix zj#t_>l>bl;XD^IF$g$HT?!BL0H6Q1S_=}@xw&ervlyV0*Zhrt8M9t8CsGKgVb8vYs zOM+j*Ff};~TEPnN^;}C$65d!)yq3!!kVM|`1t_tVN2N#O~VwvwGWj{=2OQhJ~g3B8lODRVn8gfn>Xu^xJP)t%rqU8Wr{qTTed5{TK3&zo%@twRxb`^A< zu!1{O(M(T|ZsG1%#utuN-`02tb|5O zDpUN=DKPsH3Vx$56fZp>FTRozj1Ru2&@*z_W>!qM`g~}ck`Ky{-=NhWO>ooe0!h}( z;@pQ!%u|j+MBh9z?6E}Eg2~7nRzqv(6{=d2i^kDxD)U*FH>h4fb;4SZDdcgSS}eXD zxbTW5FXEj|qsECb!jIqk8m~PG~ zAcJZc6v@ zS`=Q=JmYz=417xRKgwxfQ4Cg3=fUp}VNK9C($9ZQMGqMjzVj(g`5S%y@RDNAl+a&G zA)Gy_g{6-Zu(oFuok*C5l@7(cl8rsIHMx$LG;%rJ+5CjWJo9MAfurP}xQb5Hui%82 z1mN`rZ>;iA|f+tzWIrqV&@wvL2>?`3+RZAsN9?(+V+uBX~J0d&~lJ*|}F zui-ZI1!8t`BwWINlFbVr+|T>TJ#hkpL`EW~cOI^IMxf4CnL948gn15uQ0?)>^Mi}A z;inh1mz<|ByM<|kkQ$8gX2M&6bzLl#alXr&S_~BF)Tv(1d(Shv8lDEB&jI*-+!=;4 zk4R&u6wYo|gD(HvLrzb2HHAji@;q|yli783NPL-wi>Jfz`@upCBstJ7cHZu&`#>#u z9Bw_!g7@$gtYGb$%Rgq}zdk<13v?kZD?%xz;t&!WfkeIvER3E}xd9J*r+eaP>STme zyrorLQeAzM|^BVcNuxsh1yTJGi2X?%2QWnzos07*9-=Y>4o=bT}0 z4l$P!lDe)7-?mY-VTv{`4^&XYpWCz`A`l8}DsK8o1oJj#!YD}{12e-BFk&4k=O{xu z-5y6<2dOi9JZ2`gP_mmIMBHppdd-CsE1yGs_IBuN<7nfSTza1#ie*#y(J(hB?9AtI zB-$HW`WC_Ok^s#-dXTo~sw4f%3tFo+l`fq6L%gRGaqyxzHPLZ`l0jPz95yedy1Hn3JWCX(a(XD)=M}vRvLkgFQH+xRk2{=t zi&FXfop}6`9ttm#fQe%!MrdrH+=w+a{oo#|a63Zo!M?OLcQp;zE#u<9%OP!yC#0tQ zbr#s5hbJFhV9{}qax@Q6*?2`VDY?&ebeB_5^8$J|`YYx9+(XuLI20dC$GrkK$QQ)H zq0SXQR=04S>s=7DA`$#24fW(!x`8qjW05NSnB?70lIwE9>M>`y?d4vm@>7A@jn}ln zb~FOJf#Kts_7zslUjCSk>YeiNEpkN4WIjSaUmzL7E;G3oU&#? z{OT+&cf&<$Q;xucBlqZPPZlZ9>ZM>QHvX>(N65vSRQByA%^AbSz`tu)BUBL^#u;Lt zN;i$I`|LcUj8IV8Li2CtBEu^NZKK{gtUUdjn%teq$8|o|m+}wL=!5}M-CIKjc5G6f z&EvM1&BmN@YdNho7imk!7<72Exll_y{ZSJ_2ww;vyV%^g={(s~TOstZDUMjqfZLz~ zsYVOJ?a>FSz8#IBb+2i{%p%HDNu~8B19XqkUzT;vN+nUZq2A&$*BnVw^>d9VEUjq{^ai zbYzV!G&X92uVNR4Uu>k)9=(HjJEU>$lsP_%A0qX8WjvQPemGv64#_oRAXmsny`^Dj zH2B2p$%sYD>?F!F{6(8p?XiLlHv|rZ!Jsf2ftC(vtPq3b_f!-nI^sm*XF9yb6{~%J zvCl;S4#;@IOwACo|NSIep%OkBuAYNtja+OlGlt;^a8W;xQgdrCmY8Y4-gYX)jK9#I z(}C1#`jEovGH7Y@etOb(gzS?!l!Q$~{exk$)-}VGmEKqrd4Mc;>%rjMdvd$Cm%#G*!u*Ym<|Dfq~b7J~9lJDe)(A`5FL zY%A8nSM@nmIW$0xQ3M1D59|WEKSb*LCw34Q1LTej8QPg z_gw=#6T7YS`0YlL_+^ET2aD)6*qYPkAsD06tJFN}^u zI%s6yC-^KS&Bue5MmqF{kHW)Vw0>0`sa?;dZokF2#rl2|dX=eTvJb|79*?heZjkO5 zgPG_qhl{Vg=(L$UTv=;=xH1?HnIpM`HS$=J5tDwTIV!C zLH5f?Y`PiwF5JNl9Rs&fv>1w=U`ddMQ+o*KrE^k z#SQyZtd%3UyN$$l_YqJqzeQsE-Rb_>ARuCx4*lLtey{7fBLP#Le1Co?gWVJ9+chhg zJr<%O&JLHJEk>GTJZ{d-#LowFx%+W3IAT{v65{Nl#b3UZJibfAR?`nR1kI6ncs9OD zsXIS<8H+xZT6&&liES>sX|oRx6C_8#Vyz?b2dcQ^^Ka6GxlRK8ELe~ zVe{i9IL+1>eT|ldWr{N`IHHJYF~!_dIeA>}(Z=*lHw>5U;f*-qg90HH;$K=YAANh- ziV6E(8WA`ig&+GlbN@n$PYJ<;&7Vm7ydQNnO~K6$ZAgrY=9cmD$a6pwPBKntlC381 zDiMl}ZK0JZEj+mN(Uy@TxE}>`5mPk>nimYvGrySJ2p`&fwdIX?6Zx36xGaa`3=VPV42fTVx;B{vPjX4kp-6PY; zjZM@_Sw}>B+(Jy!O2ekwafsc*!_%m-cwmx>cSS1LYQL5%T4O|tx+7>utTPH^rExkU z3Q6CpN&o3Z+P6~+g=u^_6zx_*@TxJ`m^hL)I3-brnlthy4bg6$B3jIw2$79p2rRFn zPp<>1%bT@M+NJUA2J4izEue)B_$l7zakru19R5WSa-DAuW$Q#`ca`OoA4yb;+%g)=8{ zZ-e-B(BKR$PpcqBo+w4lI!C%j(@~dELOJrUXkMKmeYyFD_}u+6sxCVzYzTgB_N=2n(FsPqTW*w3Vi}Jr0|Un+fRma+GH-s&fGpNe^j5V-ZI&PP>0hE_aSbG1-$aIq_E6ok7&E%P% z@f8yw@t#e^Bxk~HY$$dqgy6s;d5Zm(&wc7SO{xx)VX|qM6s`#1r|Eo*o+yHP*Ki24 zd&19FF+ApEaka`Gg^~hX?nXuIFb~JiEtkkSrJh&B`^35<(ir0tg5w?vDE%*<6xbI_ zP4}IrP&d9T4NTW!&uQ8eaNCMqBt~QAU^X)3XYqtZG!cAv64gxQp}(V*yX(J+nqS4? z=dduIlnP?3<7awrng+cm@-#y$2{(=fBe;J9iQZp`w)cN&X@3oU7B@z1q%K)+n}#h- zZ)r@c1ZBG_A=Z5ZjVuemYW@^yghaYizGV>9^?uWCNe%iMDT;*8>{}65^KkR%bm$7? z;Jt?)=bAl0+jpkp*-BNMtYIzsVsi{zZ>HVn%_;4WGzNQY(Hp*yS0STJA@!j+y5SFn zZ~H_o!W@*Wb+{cDbE)N}6P(s8qj9FHDDwVJ{H3*5DPxx(C3bnCeeP=NEVDs>?|WL9 zDT5W>FX&XxClCKiU%u1jaA*ESuZ;_GSBH6^Usq662^KRo9PUDO&Mh@E46 zaJ$zWyoVx)RqNyS=eKg-_Dv;6r2>-a)kJsyJUBULb8|ddH5K!OPajf0&;kE+iVv~j zaRC-2)3Kf!mIsg?ca|1>55dA{KRj~YO|gcN;Lk9J>^^(EU38Wv*sy218`U&tVh;4( zv(Yg(fGV}bFtl8jOxgVBo$+kczH(+u!T#gw^!=dTe2ya98%fK-9Y_1)5dWfpyuA3v zc>XH}<7dspsz7~Y48@YJ;(0Pw(uBaTILsHypw+*;(Oa>f^GS)M?|t`pw_bP9?fbf1 zQu!*bpAEC0pAy9)c6H;gSxY@DcTr{gT$(y*2RB=1nAaMVM14!I(Zqxibl7PYHcTW` z7Ks7w>FAxdfKp;MJ7dsL8nQ=i(6Z(w$w#s;8#bBYR?A*`A*e^jRU5ep&&N=)|7*&c zIE{TX(v$wjq36E`L1mGDH^+ZN&;PwuJ#u=s2!E#D_6IF9?XQNmWt_1t|JE`K!eMvv z>}>WbwkCQ^3~MhsV{_29b+-SCu>1S%Y>!TDO`3K!>|xCr+v5vb=R^yKKW?(nu zS-gJj#%t*d2lg!*crA}M(sh>V2lmt7rg(jQuDWA+Y~aA>F0bz&8+UyFG(i8s;!Pl` z=`vFW53(70Gn1QiNh*WMYg4^hCDimdoCXi^cYBkRoAkKi22(b%`moc~^m&Q~QzZ?3 zI2@bwDGh@Rxl|v{05t=l@xe6JZXd4LCWDPjgXue1eYqKGhN6_A47#zQFHd2UAysAQ zuyv|0U$q)d%4sOmx!adg-$dIMH*~~{)lZ;D&1iekP*#wkpU_y7kxIkR(TG$(;dwP< zjq#!E_-?TWqtr=hATT zE!IGJhPs6>Wu&CnFmQWevqhlFNa;Xopi;HEWvJ6g*+h4sN`13sc-+YG$E-nWJ?d6b zMI$E`4TChsnyum*Mozv>4bqxdx86HGQvSI+Nas_t^?{|4Q$Oge!Ma2Zn?sb*3N~7> zKDosvO=a}-T1K#;gobUV(`Y4sPq2}4i*0t?=$TE#5EGh)U0%^>l_V|1%(2C;sA05P zjuB!JpkZG&K6+NQC&VhY#lC!Lv}Ok})P|wqP)Qj(XG{yVD{OH%t1@=pni1+yt>JjV zY3zb?PpDIUizEF?+}K4g;x3mS4X5ixW0!(xyWGZFoEjR&E=MqSxzB4jw~UWniSOCv z`KiVE=F(U#ofzg#)O6{jj9*Qsh53?OUGAuiU&~>H`AcZJ_BoATFYXBoRBmNegmUSG@D9TuRub9#Kdp|OX)J3O{^=j_sW<1J!%Btz4E zjxy2IOAC)GY;|8ynP?thgvV5Cdc1U+Xqo5coR2)6N*G6^k{m%FPdmGU!+Cs z9c%SmYM5w$%ZS)Fuj#csK5_GNPsD*wtzO@kCT{&8Mjj+;c{B4*-exn3JVb8uCaF$# ztWAqdmC*8`b2v|S^52O}Q*QI&il6M-MB0-<)AHphp6r%1+LP(n=1Xau?2${`lNF%l zCp0m6NA=F0?ASKHjh`obcaWlvF|_otU%A?IBk2(<7%yvFB*o zp?1a7qcujc7aZF|of@adYSUsb1!(Vbo0uMNyc2sRwtbi9=jn-Cq`0dL?J!^dnaN(G zxa)=OVS#kjnW=%axEs~lyF;C4rYG*iHPpB74v(Lic}$9L>d_95DxP_?XcXTv)*c?$ zIP=rnwD`7p?TEb-Gqa!X#NYhX9&zCF%;O)Vgxf@&$V2>(p0F7wbdqnr>m+j#s4ow) z{OuJbf&%P5etuq2J~07gW=gp*Wub(LvRuMMr+%GeroLNb!s|@__^bKPe^LMI*O^>Z zB)2G9kY98D_6GY&!H*N1oACC1cD#pw6$^pz2HFC}3#|!115M;D$QR+i!S7=t5Y7<@ zbOHy$W|Uu{Tv*r()q?7x`U<8{;ZuP-`lHaZ&~fZ0z^}sm8lVD10xSN=f%ZV3fH~~-BcH@E z0icov_x~6MvKX94aR7cN7D}N5&^BlYdQr@!;APOQg@1*v0eS`bC43?1!<-7;0o2f& zA$!1^0zvE^fu@6<=xKl*JI*Y)|5;ILqi8|V4n84s0w%EXD-{&t0ve&s*t18!fINom ziF^*e1HKDh6I{joC*&yjyC4I8HFkbM?*V@h%pyA>U!%i_pg0M&K`{)zf_@x$0G&P% zM81Ta4o`sp1(F2c4_^nr4!#e{jh%TAf!-Y17ri6=4E(*7UF2B!U!kADX+Xb?!UhLX z5RPNX4_*>oB~V2kML!A3fDpQD%%|Wl!)t*I>^uM)!B3d!V0Hk#36z0O4f+CV3l+zl z9o`Oe7i1Y`JpMOf!oXn+<}gTxW@6C+ehvEl&@yliT@Ge!Pz|sL{swdhcJiSV?EH*w z2tEWI3*do&i+&j15bA}z6}}qmMV4K8{;xq9jRQBJki#Gq>_@HzYr$Rg`q;s#QwZgt z8C(GEn4N|$fkhm144DY!fhuGF9JB*$Lnn_s0j|Oa0uFdC%#!IaE}#s-7TO64;ES*{ z2t9dgXFJ zE;>)-NKl5o0CbF%=YKc`F(^1uL_!mhov>7d+<~lwtPWj;t`o3<%jmMuk)Y2&68tRo zw*y6ZSLhk^_pnQb4}-snUK*IeKLmN0QJC=f2g4X(ppJ4DsDNp7A8_DS_-%j}jAF+d zohUek{2VzEel;+~emuG-(DTUO;A_BD?5sy81SLVOp}R0Y4*w4GeJjuZ^FRj!78F%b zF)#^#82wiGGf)L+3p!sM+zr1D7^0s>M}gmmz88KyycqfgWE0Si>;?UZnJ9b=R0DHw z_*?LFVf?WP>_c%0x*o{E4`AsCbQ$?RF5n`(KGYw52y_tY4|RjaVkQmmiCreW4;->gq;S!jXVVHL$`h9@t?-xUs2G2Cx(LHAv!te55N!j zup9w>02Ht{f&5o+2758+27v|qMa*u4Y2@3O&%^%;uLGUJd;?S&+ywKOUtalW3_vu7 zN1$7<>;|^O2Z8J8>#=MF-k{HhZbN4YWq~fDy8-_l>_OIoasYf>ghM#K0vN*=p)Dr1g!&lAPSU?XPD$l35eu*(Jf z(WQYVa1FB{s3Yb#z$E+^=y|9lW&+THl@7Z4EuLB?WpTI8oam+&DOQ6!o+PDw_U=9Bn z@M0DUr9w|YbubHne+wLOOdB*4T?%H`;oDdK{a=m26BL}#a+KxpJWvl1hfW;_5TU;y z@5XFDQ~<1kUyB(D%)qy$VuVPwe+s`R8*)xX?vXMLOBV3%om5ZKD(ZOG@-z0O4ZbT^NP{7@<{mlhL^ z&(1=RMs{*frcvC&I%xc!`M&X#{`DP6c+Xu0whgh9Og$SC zo}Xr0mGthkK4Jf_^uMr)B(ra4-^{pL!PuNG^4?H1Q>N2IEL)4&RGi&p`znb-m#9_L z(qP{^)Dtmt_arM)MfE?pTy=wXM`RnHh~DLG zesO)Z+_sN%_vLQ0ian6$<+jgK=oQK5+|(!2r7QM8>DdFt!DHM<=}IFu_D7Z1x~EvG zOow$@s?P3Xu~M7M;^kDIFY)PDe^I5FuHka2%Zl(zmRN?=o3>|$qVFtta+KdcFtFBM zdc@1F^X0|JZJnyS(1_u(@#5;E=V)8U}Y-dJb6A+fvHScFlPO%=6d9ySp#bjUT`s-!<8E%{}R&1~zM z3OjLT);b<@!TN$^bGZ(_d<(@PhfG=JsTf|XFVCxZuYqWvk zU}q*P!EFCNc%;DIT8~`lU~lVK=(tNHnk?WFP)#Q8+%s0_?3GNW7rCrHN~7oocx*rH z8g!bd=N5XohrDxl^CyZstw&gweh3`)3c0(Xw{{0WoypS3d%yWF5hw6b)}uOGwgb-#B|t= zvA~^bjkaksUd=x@?9^{lFw~2<`Hkcr>BO!oyyq){#hh@_#P-CV-p$1)qSmX|^+r8< z7fp_SxJKbv^guPUM~skzxCfbgpMOtGm5uYs*bN?^?~$~;i%+sr>A@2-agSmbe~N#? zs7XtpWTiX1UnJ;zCN6x5_B4A@?OeR~)qbbs@YnV6rV?-3ozuhK^+*b=Nj%t@7x1K= zl(X-Hf|1w$r9$e|{$J<$85@^>>GfJq|G4;=ZcPxV2w>*Qpm}xL@+C_XcwD@rNE`18 z9%8?cTXBdiJIN|RR(L_zP2u7qjizw(@SINNwKKWTMs^eEX6QFFp33u6j2^~vB|q0q z6C%|=;-nkmH<6bnsI?k3NfuTVJ@JMAV-KU1cmDf0;3qdWsLJdXk`?tSyGORhn{ys(wYUzs<>+vH6bsfR33jb%deCw7nDn69 zb8#~Fd9pNZ);=rulZWH&T+7WNy#}V04a}@&+wS=vJg7YNIMVdS;`4kPqw}+lcC+%; zd_^{{)rHO%7!o{O-|4!#M@21)B$6~7s^>l4J?QBza(BC9Q|!7|eLyC1|A%U}N6CWJ zT!+KwJ4*Z)2!rCIh3e_Tvn5-nOFx`$9?S~4pq|Fxaw*Y1DvZMIew@Cc>9D2!I-?KN z6HVQvB_|}=)obk1xJ@NVclnH7$6|_e#Z-`lq{AK;C*2;>$X@#G7eP$)H?Gbyx&qar70DbLTTTm8md~6{ z7cDziIVd}AX;rnmSmcbY^2^LKIr>6YRe5_Q^TnzyrPf!!uIf%n@re>yJl@i58{%Jd zVYH{ZWR5B5Y*Mhs^jUg|g#)$5+`}cX<^tj89{KVsj*G(T2Q)f8D@6Thp{FzVPMxbN z3VrDzej=3j0;NIF<-&z4eWAYDI`)S*2s82wE=shR`RVxE5s2H@v~#?wyw+!C!eTN# z%Bx*hY4Xa+=aN%OkmK@Vbw>Si$SZ@-0kIb^UU^b>@yeE>4o>Q^Y5FtX%+}5xY1{oD z%?^7L)a#rIZ|0RztKFz7gUDbo@i#JzL}s7pj)m4J;c`ZrI3v*1bV7(=5s9?UAb7@!sM- zxqi&*eSW#%N+~3;syqDN06t}BLdvEdH4Q}IkGGih?u3&#C^^(*dV%JH{ak&@@T;@(0xTjy&EL1vF?0*Xlxtbr;O2L1>NQ`>DmqS^QBUUuiRJ& zKl0WrG$bd3YdSOh_RZ&+cJD}(T3r^G$o{NS$IWlg&nGI?mUthXJ3!VFekn)HoN-e(%EkPuNu1MUN`QJM`V5T9zaCEqal~Y8dDd&Xsb3j)%ySvd^tD$8u;_L^jTt$#v(-I~+pZwUO2qEG4d!*32= z)8XRCy=En-%%@8ps7grQL7wdkAN4j!a*IUq9Y_GAw<@t@SX?@a~_qGemw#t;t z%DrM&bgGD)ba|k%#VxJ&z}+^zD~iU(>Ai(+F znUdy0U9M-LmLJO3OA!5Er&{GE^I4FYfBQBtyYR0zUI%dc=#{UGC}g33eD@>!-$nv| zclbW@-~AsgzQ$U;1z(I2%7vMj%7qEkb3fr0{eGTVK>@b>xdh&jyhkk5xfvGWKaTpt z{hLeum#3Qj(`{$Pb;bll$Nb}M&-Y9eU!D2{H-Pi^yX0u%2GZHbEYzjh{HuwSwSPYE zEOY)?qBxrbpGDR>VVV5ziD~~Yr^u!LyR+a5{^{0G{~PZp{ZD>~@E`sRVf{ay>G%Kq zO#7z~=N~=|@*gKqZ2v2t1Jr;2*lzvPy-68<#DX83P}wO9<1G09f}o55Dbgq0-zQq2 zrA}-AAWMGL0iOI?`nr6% qm?l_+TE<%0o2l#R<&S1GGa1%pA1-HN+LSMQM2Ps?wdPwM5&0i- + quay.io/minio/minio:RELEASE.2023-06-19T19-52-50Z + args: + - server + - /data + - --console-address + - :9090 + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + strategy: + type: Recreate + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 +--- +kind: Service +apiVersion: v1 +metadata: + name: minio-service +spec: + ipFamilies: + - IPv4 + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: ui + protocol: TCP + port: 9090 + targetPort: 9090 + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app: minio \ No newline at end of file diff --git a/examples/python-helm-demo/minio.env b/examples/python-helm-demo/minio.env new file mode 100644 index 00000000000..b19ec5083f5 --- /dev/null +++ b/examples/python-helm-demo/minio.env @@ -0,0 +1,7 @@ +export AWS_ACCESS_KEY_ID=minio +export AWS_DEFAULT_REGION=default +#export AWS_S3_BUCKET=feast-demo +#export AWS_S3_ENDPOINT=http://localhost:9000 +export FEAST_S3_ENDPOINT_URL=http://localhost:9000 +export AWS_SECRET_ACCESS_KEY=minio123 + diff --git a/examples/python-helm-demo/online_feature_store.yaml.template b/examples/python-helm-demo/online_feature_store.yaml.template new file mode 100644 index 00000000000..7acb9582c51 --- /dev/null +++ b/examples/python-helm-demo/online_feature_store.yaml.template @@ -0,0 +1,7 @@ +project: feast_python_demo +provider: local +registry: s3://feast-demo/registry.db +online_store: + type: redis + connection_string: my-redis-master:6379,password=_REDIS_PASSWORD_ +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/test/feature_store.yaml b/examples/python-helm-demo/test/feature_store.yaml new file mode 100644 index 00000000000..13e99873ee7 --- /dev/null +++ b/examples/python-helm-demo/test/feature_store.yaml @@ -0,0 +1,7 @@ +registry: s3://feast-demo/registry.db +project: feast_python_demo +provider: local +online_store: + path: http://localhost:6566 + type: remote +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/test_python_fetch.py b/examples/python-helm-demo/test/test_python_fetch.py similarity index 73% rename from examples/python-helm-demo/feature_repo/test_python_fetch.py rename to examples/python-helm-demo/test/test_python_fetch.py index f9c7c62f4fd..715912422f3 100644 --- a/examples/python-helm-demo/feature_repo/test_python_fetch.py +++ b/examples/python-helm-demo/test/test_python_fetch.py @@ -1,6 +1,7 @@ from feast import FeatureStore import requests import json +import pandas as pd def run_demo_http(): @@ -14,7 +15,14 @@ def run_demo_http(): r = requests.post( "http://localhost:6566/get-online-features", data=json.dumps(online_request) ) - print(json.dumps(r.json(), indent=4, sort_keys=True)) + + resp_data = json.loads(r.text) + records = pd.DataFrame.from_records( + columns=resp_data["metadata"]["feature_names"], + data=[[r["values"][i] for r in resp_data["results"]] for i in range(len(resp_data["results"]))] + ) + for col in sorted(records.columns): + print(col, " : ", records[col].values) def run_demo_sdk(): From 1119439c49bc90e62f02da078901509c1d740236 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Thu, 19 Dec 2024 08:20:57 -0500 Subject: [PATCH 43/90] fix: Fixing some of the warnings with the github actions (#4763) Fixing some of the warnings with the github actions, most of them related to deprecated actions or libraries. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .github/workflows/java_master_only.yml | 4 ++-- .github/workflows/java_pr.yml | 14 +++++++------- .github/workflows/lint_pr.yml | 2 +- .github/workflows/operator_pr.yml | 2 +- .github/workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/smoke_tests.yml | 2 +- .github/workflows/unit_tests.yml | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 2475321f706..b7f49d14544 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -72,13 +72,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 0391e6fde9f..3aea4d275e8 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -53,13 +53,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} @@ -97,11 +97,11 @@ jobs: python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet @@ -137,18 +137,18 @@ jobs: with: python-version: '3.11' architecture: 'x64' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml index 81732258455..33fafdcd23d 100644 --- a/.github/workflows/lint_pr.yml +++ b/.github/workflows/lint_pr.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v4 + - uses: amannn/action-semantic-pull-request@v5 with: # Must use uppercase subjectPattern: ^(?=[A-Z]).+$ diff --git a/.github/workflows/operator_pr.yml b/.github/workflows/operator_pr.yml index e4d371b9454..232ccf7d339 100644 --- a/.github/workflows/operator_pr.yml +++ b/.github/workflows/operator_pr.yml @@ -7,7 +7,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21.x - name: Operator tests diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index e6a9e3e8bde..2825b96f482 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -45,7 +45,7 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 9a898dd4c54..a7eb1966269 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -31,7 +31,7 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6f46d129638..443f40270ff 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -33,8 +33,8 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache - run: | - echo "::set-output name=dir::$(uv cache dir)" + run: | + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: @@ -52,7 +52,7 @@ jobs: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version-file: './ui/.nvmrc' registry-url: 'https://registry.npmjs.org' From 330ba0c6a6e2b126a6b41ec0ffb9ac8c871d3fa3 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 19 Dec 2024 09:13:21 -0500 Subject: [PATCH 44/90] Update README.md --- docs/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index 5e36e1ce40a..36c83ed177a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,12 +42,15 @@ serving system must make a request to the feature store to retrieve feature valu ## Who is Feast for? -Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams -build a feature platform that improves collaboration between data engineers, software engineers, machine learning -engineers, and data scientists. +Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams build a feature platform that improves collaboration between data engineers, software engineers, machine learning engineers, and data scientists. -Feast is likely **not** the right tool if you -* are in an organization that’s just getting started with ML and is not yet sure what the business impact of ML is +* *For Data Scientists*: Feast is a a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. + +* *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. + +* *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. + +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. ## What Feast is not? From ae2a521c317e96490935609ceb643825266d6bd6 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 19 Dec 2024 09:37:10 -0500 Subject: [PATCH 45/90] chore: Update quickstart.md --- docs/getting-started/quickstart.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d35446ce7f0..a83897005fd 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -10,6 +10,9 @@ Feast (Feature Store) is an open-source feature store designed to facilitate the * *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. + + For more info refer to [Introduction to feast](../README.md) ## Prerequisites From a73514cd4f7fecbc89679566e0f8a0af16b6b06d Mon Sep 17 00:00:00 2001 From: Daniel Dowler <12484302+dandawg@users.noreply.github.com> Date: Thu, 19 Dec 2024 20:31:58 -0500 Subject: [PATCH 46/90] feat: Added pvc accessModes support (#4851) * added pvc accessModes support Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * made accessModes doc line more clear for users Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * Added multiple accessModes to PVC accessModes test Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * function for pvc.Create Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> --------- Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> --- .../api/v1alpha1/featurestore_types.go | 2 + .../api/v1alpha1/zz_generated.deepcopy.go | 5 +++ .../crd/bases/feast.dev_featurestores.yaml | 37 +++++++++++++++++++ infra/feast-operator/dist/install.yaml | 37 +++++++++++++++++++ .../featurestore_controller_pvc_test.go | 8 ++++ .../internal/controller/services/services.go | 2 +- .../controller/services/services_types.go | 10 +++-- .../internal/controller/services/util.go | 30 ++++++++++----- 8 files changed, 117 insertions(+), 14 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 84b4d8e841b..2eb9ec8554d 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -242,6 +242,8 @@ type PvcConfig struct { // The PVC name is the same as the associated deployment name. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PvcCreate is immutable" type PvcCreate struct { + // AccessModes k8s persistent volume access modes. Defaults to ["ReadWriteOnce"]. + AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty"` // StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value // means that this volume does not belong to any StorageClass and the cluster default will be used. StorageClassName *string `json:"storageClassName,omitempty"` diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 6cba8e59234..3241dff775b 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -524,6 +524,11 @@ func (in *PvcConfig) DeepCopy() *PvcConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PvcCreate) DeepCopyInto(out *PvcCreate) { *out = *in + if in.AccessModes != nil { + in, out := &in.AccessModes, &out.AccessModes + *out = make([]v1.PersistentVolumeAccessMode, len(*in)) + copy(*out, *in) + } if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 7fbd38ed31b..fd8861cef14 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -254,6 +254,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -622,6 +628,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1006,6 +1018,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1506,6 +1524,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1879,6 +1903,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2272,6 +2302,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 73abc3717b8..435be789e51 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -262,6 +262,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -630,6 +636,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1014,6 +1026,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1514,6 +1532,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1887,6 +1911,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2280,6 +2310,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index fe0caa38e63..e64e5cd6245 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -69,6 +69,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { onlineStoreMountPath := "/online" registryMountPath := "/registry" + accessModes := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany} storageClassName := "test" onlineStoreMountedPath := path.Join(onlineStoreMountPath, onlineStorePath) @@ -85,6 +86,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Type: offlineType, PvcConfig: &feastdevv1alpha1.PvcConfig{ Create: &feastdevv1alpha1.PvcCreate{ + AccessModes: accessModes, StorageClassName: &storageClassName, }, MountPath: offlineStoreMountPath, @@ -162,6 +164,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.Type).To(Equal(offlineType)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(accessModes)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(Equal(&storageClassName)) expectedResources := corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -179,6 +182,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(onlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -198,6 +202,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(registryPath)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -283,6 +288,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) Expect(pvc.Name).To(Equal(deploy.Name)) Expect(pvc.Spec.StorageClassName).To(Equal(&storageClassName)) + Expect(pvc.Spec.AccessModes).To(Equal(accessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOfflineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -313,6 +319,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { pvc) Expect(err).NotTo(HaveOccurred()) Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOnlineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -343,6 +350,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { pvc) Expect(err).NotTo(HaveOccurred()) Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultRegistryStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index f85597e648c..232e3e58743 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -450,7 +450,7 @@ func (feast *FeastServices) createNewPVC(pvcCreate *feastdevv1alpha1.PvcCreate, pvc := feast.initPVC(feastType) pvc.Spec = corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + AccessModes: pvcCreate.AccessModes, Resources: pvcCreate.Resources, } if pvcCreate.StorageClassName != nil { diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index b7c0f5f048b..b9e1a9d9d75 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -20,6 +20,7 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" handler "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -80,10 +81,11 @@ const ( ) var ( - DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion - DefaultReplicas = int32(1) - NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" - ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" + DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion + DefaultReplicas = int32(1) + DefaultPVCAccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" + ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" FeastServiceConstants = map[FeastServiceType]deploymentSettings{ OfflineFeastType: { diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 631709d6ba0..92ee2b5752e 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -92,9 +92,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { pvc := services.Registry.Local.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultRegistryStorageRequest) - } + ensurePVCDefaults(pvc, RegistryFeastType) } } @@ -116,9 +114,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { if services.OfflineStore.Persistence.FilePersistence.PvcConfig != nil { pvc := services.OfflineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOfflineStorageRequest) - } + ensurePVCDefaults(pvc, OfflineFeastType) } } @@ -141,9 +137,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { pvc := services.OnlineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOnlineStorageRequest) - } + ensurePVCDefaults(pvc, OnlineFeastType) } } @@ -182,6 +176,24 @@ func ensureRequestedStorage(resources *v1.VolumeResourceRequirements, requestedS } } +func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { + var storageRequest string + switch feastType { + case OnlineFeastType: + storageRequest = DefaultOnlineStorageRequest + case OfflineFeastType: + storageRequest = DefaultOfflineStorageRequest + case RegistryFeastType: + storageRequest = DefaultRegistryStorageRequest + } + if pvc.Create != nil { + ensureRequestedStorage(&pvc.Create.Resources, storageRequest) + if pvc.Create.AccessModes == nil { + pvc.Create.AccessModes = DefaultPVCAccessModes + } + } +} + func defaultOnlineStorePath(persistence *feastdevv1alpha1.OnlineStoreFilePersistence) string { if persistence.PvcConfig == nil { return DefaultOnlineStoreEphemeralPath From 35d6017353bed0091967aecafbcaeb15399fdb33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 04:39:37 +0000 Subject: [PATCH 47/90] chore: Bump sqlite-vec from 0.1.1 to 0.1.3 in /sdk/python/requirements (#4738) --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index f2b48d73624..fff0993b1d0 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -897,7 +897,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index a9dceac08c9..4dbfc44509b 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -888,7 +888,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 556e709c20a..7357f3741f1 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -905,7 +905,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb From c15f4ccafda18b68cdda7f958d23397cea490a92 Mon Sep 17 00:00:00 2001 From: Daniel Dowler <12484302+dandawg@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:42:56 -0500 Subject: [PATCH 48/90] docs: Credit-risk-end-to-end example (#4630) * credit-risk-end-to-end example Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * incorporated review suggestions Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * added cleanup notebook link to readme Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * fixed seeds for timestamps; other minor updates Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * fixed README.md typo Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> * updated feast data ingest order and simplified Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> --------- Signed-off-by: dandawg <12484302+dandawg@users.noreply.github.com> --- .../01_Credit_Risk_Data_Prep.ipynb | 757 ++++++++ .../02_Deploying_the_Feature_Store.ipynb | 801 +++++++++ .../03_Credit_Risk_Model_Training.ipynb | 1541 +++++++++++++++++ .../04_Credit_Risk_Model_Serving.ipynb | 697 ++++++++ .../05_Credit_Risk_Cleanup.ipynb | 296 ++++ examples/credit-risk-end-to-end/README.md | 39 + .../credit-risk-end-to-end/requirements.txt | 6 + 7 files changed, 4137 insertions(+) create mode 100644 examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb create mode 100644 examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb create mode 100644 examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb create mode 100644 examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb create mode 100644 examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb create mode 100644 examples/credit-risk-end-to-end/README.md create mode 100644 examples/credit-risk-end-to-end/requirements.txt diff --git a/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb new file mode 100644 index 00000000000..a345ec8ca46 --- /dev/null +++ b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb @@ -0,0 +1,757 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a52c80c4-1ea2-4d1e-b582-fac51081e76d", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "576a8e30-fe4c-4eda-bc56-9edd7fde3385", + "metadata": {}, + "source": [ + "# Credit Risk Data Preparation" + ] + }, + { + "cell_type": "markdown", + "id": "1f3fbd5a-1587-4b4e-9263-a57490657337", + "metadata": {}, + "source": [ + "Predicting credit risk is an important task for financial institutions. If a bank can accurately determine the probability that a borrower will pay back a future loan, then they can make better decisions on loan terms and approvals. Getting credit risk right is critical to offering good financial services, and getting credit risk wrong could mean going out of business.\n", + "\n", + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we prepare the data." + ] + }, + { + "cell_type": "markdown", + "id": "4d05715f-ddb8-42de-8f0c-212dcbad9e0e", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "6fba29f9-db1f-4ceb-b066-5b2df2c95d33", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8a897b19-6f82-4631-ae51-8a23182ff267", + "metadata": {}, + "outputs": [], + "source": [ + "# Import Python libraries\n", + "import os\n", + "import warnings\n", + "import datetime as dt\n", + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.datasets import fetch_openml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b944ed48-54b3-43fa-8373-ce788d7e71af", + "metadata": {}, + "outputs": [], + "source": [ + "# suppress warning messages for example flow (don't run if you want to see warnings)\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "70788c73-144f-4ecf-b370-c5669c538d93", + "metadata": {}, + "outputs": [], + "source": [ + "# Seed for reproducibility\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "cfb4dfd0-f583-4aa0-bd39-3ff9fbb80db0", + "metadata": {}, + "source": [ + "### Pull the Data" + ] + }, + { + "cell_type": "markdown", + "id": "3c206dfc-d551-4002-ae63-ccbb981768fa", + "metadata": {}, + "source": [ + "The data we will use to train the model is from the [OpenML](https://www.openml.org/) dataset [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31), obtained from a 1994 German study. More details on the data can be found in the `DESC` attribute and `details` map (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "31a9e964-bdb3-4ae4-b2b4-64bbe0ab93a3", + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_openml(name=\"credit-g\", version=1, parser='auto')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "58dbf7c2-f40b-4965-baac-6903a27ef622", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "**Author**: Dr. Hans Hofmann \n", + "**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)) - 1994 \n", + "**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)\n", + "\n", + "**German Credit dataset** \n", + "This dataset classifies people described by a set of attributes as good or bad credit risks.\n", + "\n", + "This dataset comes with a cost matrix: \n", + "``` \n", + "Good Bad (predicted) \n", + "Good 0 1 (actual) \n", + "Bad 5 0 \n", + "```\n", + "\n", + "It is worse to class a customer as good when they are bad (5), than it is to class a customer as bad when they are good (1). \n", + "\n", + "### Attribute description \n", + "\n", + "1. Status of existing checking account, in Deutsche Mark. \n", + "2. Duration in months \n", + "3. Credit history (credits taken, paid back duly, delays, critical accounts) \n", + "4. Purpose of the credit (car, television,...) \n", + "5. Credit amount \n", + "6. Status of savings account/bonds, in Deutsche Mark. \n", + "7. Present employment, in number of years. \n", + "8. Installment rate in percentage of disposable income \n", + "9. Personal status (married, single,...) and sex \n", + "10. Other debtors / guarantors \n", + "11. Present residence since X years \n", + "12. Property (e.g. real estate) \n", + "13. Age in years \n", + "14. Other installment plans (banks, stores) \n", + "15. Housing (rent, own,...) \n", + "16. Number of existing credits at this bank \n", + "17. Job \n", + "18. Number of people being liable to provide maintenance for \n", + "19. Telephone (yes,no) \n", + "20. Foreign worker (yes,no)\n", + "\n", + "Downloaded from openml.org.\n" + ] + } + ], + "source": [ + "print(data.DESCR)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "53de57ec-0fb6-4b51-9c27-696b059a1847", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original data url: https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)\n", + "Paper url: https://dl.acm.org/doi/abs/10.1145/967900.968104\n" + ] + } + ], + "source": [ + "print(\"Original data url: \".ljust(20), data.details[\"original_data_url\"])\n", + "print(\"Paper url: \".ljust(20), data.details[\"paper_url\"])" + ] + }, + { + "cell_type": "markdown", + "id": "6b2c2514-484e-46cb-aedc-89a301266f44", + "metadata": {}, + "source": [ + "### High-Level Data Inspection" + ] + }, + { + "cell_type": "markdown", + "id": "a76af306-caba-403d-a9cb-b5de12573075", + "metadata": {}, + "source": [ + "Let's inspect the data to see high level details like data types and size. We also want to make sure there are no glaring issues (like a large number of null values)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "20fb82c4-ed8d-42f8-b386-c7ebdc9bf786", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 21 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 checking_status 1000 non-null category\n", + " 1 duration 1000 non-null int64 \n", + " 2 credit_history 1000 non-null category\n", + " 3 purpose 1000 non-null category\n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 savings_status 1000 non-null category\n", + " 6 employment 1000 non-null category\n", + " 7 installment_commitment 1000 non-null int64 \n", + " 8 personal_status 1000 non-null category\n", + " 9 other_parties 1000 non-null category\n", + " 10 residence_since 1000 non-null int64 \n", + " 11 property_magnitude 1000 non-null category\n", + " 12 age 1000 non-null int64 \n", + " 13 other_payment_plans 1000 non-null category\n", + " 14 housing 1000 non-null category\n", + " 15 existing_credits 1000 non-null int64 \n", + " 16 job 1000 non-null category\n", + " 17 num_dependents 1000 non-null int64 \n", + " 18 own_telephone 1000 non-null category\n", + " 19 foreign_worker 1000 non-null category\n", + " 20 class 1000 non-null category\n", + "dtypes: category(14), int64(7)\n", + "memory usage: 71.0 KB\n" + ] + } + ], + "source": [ + "df = data.frame\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "a384932a-40df-45f6-bfbc-a9cf6c708f1b", + "metadata": {}, + "source": [ + "We see that there are 21 columns, each with 1000 non-null values. The first 20 columns are contextual fields with `Dtype` of `category` or `int64`, while the last field is actually the target variable, `class`, which we wish to predict. \n", + "\n", + "From the description (above), the `class` tells us whether a loan to a customer was \"good\" or \"bad\". We are anticipating that patterns in the contextual data, as well as their relationship to the class outcomes, can give insight into loan classification. In the following notebooks, we will build a loan classification model that seeks to encode these patterns and relationships in its weights, such that given a new loan application (context data), the model can predict whether the loan (if approved) will be good or bad in the future." + ] + }, + { + "cell_type": "markdown", + "id": "a451c9a3-0390-4d5a-b687-c59f52445eb1", + "metadata": {}, + "source": [ + "### Data Preparation For Demonstrating Feast" + ] + }, + { + "cell_type": "markdown", + "id": "dc4e7653-b118-44c3-ade3-f1b217b112fc", + "metadata": {}, + "source": [ + "At this point, it's important to bring up that Feast was developed primarily to work with production data. Feast requires datasets to have entities (in our case, IDs) and timestamps, which it uses in joins. Feast can support joining data on multiple entities (like primary keys in SQL), as well as \"created\" timestamps and \"event\" timestamps. However, in this example, we'll keep things more simple.\n", + "\n", + "In a real loan application scenario, the application fields (in a database) would be associated with a timestamp, while the actual loan outcome (label) would be determined much later and recorded separately with a different timestamp.\n", + "\n", + "In order to demonstrate Feast capabilities, such as point-in-time joins, we will mock IDs and timestamps for this data. For IDs, we will use the original dataframe index values. For the timestamps, we will generate random values between \"Tue Sep 24 12:00:00 2023\" and \"Wed Oct 9 12:00:00 2023\"." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9d6ec4f6-9410-4858-a440-45dccaa0896b", + "metadata": {}, + "outputs": [], + "source": [ + "# Make index into \"ID\" column\n", + "df = df.reset_index(names=[\"ID\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "055f2cb7-3abf-4d01-be60-e4c7b8ad1988", + "metadata": {}, + "outputs": [], + "source": [ + "# Add mock timestamps\n", + "time_format = \"%a %b %d %H:%M:%S %Y\"\n", + "date = dt.datetime.strptime(\"Wed Oct 9 12:00:00 2023\", time_format)\n", + "end = int(date.timestamp())\n", + "start = int((date - dt.timedelta(days=15)).timestamp()) # 'Tue Sep 24 12:00:00 2023'\n", + "\n", + "def make_tstamp(date):\n", + " dtime = dt.datetime.fromtimestamp(date).ctime()\n", + " return dtime\n", + " \n", + "# (seed set for reproducibility)\n", + "np.random.seed(SEED)\n", + "df[\"application_timestamp\"] = pd.to_datetime([\n", + " make_tstamp(d) for d in np.random.randint(start, end, len(df))\n", + "])" + ] + }, + { + "cell_type": "markdown", + "id": "f7800ea9-de9a-4aab-9d77-c4276e7db5f9", + "metadata": {}, + "source": [ + "Verify that the newly created \"ID\" and \"application_timestamp\" fields were added to the data as expected." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9516fc5c-7c25-4e60-acba-7400ab6bab42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012
ID012
checking_status<00<=X<200no checking
duration64812
credit_historycritical/other existing creditexisting paidcritical/other existing credit
purposeradio/tvradio/tveducation
credit_amount116959512096
savings_statusno known savings<100<100
employment>=71<=X<44<=X<7
installment_commitment422
personal_statusmale singlefemale div/dep/marmale single
other_partiesnonenonenone
residence_since423
property_magnitudereal estatereal estatereal estate
age672249
other_payment_plansnonenonenone
housingownownown
existing_credits211
jobskilledskilledunskilled resident
num_dependents112
own_telephoneyesnonenone
foreign_workeryesyesyes
classgoodbadgood
application_timestamp2023-10-04 17:50:132023-09-28 18:10:132023-10-03 23:06:03
\n", + "
" + ], + "text/plain": [ + " 0 1 \\\n", + "ID 0 1 \n", + "checking_status <0 0<=X<200 \n", + "duration 6 48 \n", + "credit_history critical/other existing credit existing paid \n", + "purpose radio/tv radio/tv \n", + "credit_amount 1169 5951 \n", + "savings_status no known savings <100 \n", + "employment >=7 1<=X<4 \n", + "installment_commitment 4 2 \n", + "personal_status male single female div/dep/mar \n", + "other_parties none none \n", + "residence_since 4 2 \n", + "property_magnitude real estate real estate \n", + "age 67 22 \n", + "other_payment_plans none none \n", + "housing own own \n", + "existing_credits 2 1 \n", + "job skilled skilled \n", + "num_dependents 1 1 \n", + "own_telephone yes none \n", + "foreign_worker yes yes \n", + "class good bad \n", + "application_timestamp 2023-10-04 17:50:13 2023-09-28 18:10:13 \n", + "\n", + " 2 \n", + "ID 2 \n", + "checking_status no checking \n", + "duration 12 \n", + "credit_history critical/other existing credit \n", + "purpose education \n", + "credit_amount 2096 \n", + "savings_status <100 \n", + "employment 4<=X<7 \n", + "installment_commitment 2 \n", + "personal_status male single \n", + "other_parties none \n", + "residence_since 3 \n", + "property_magnitude real estate \n", + "age 49 \n", + "other_payment_plans none \n", + "housing own \n", + "existing_credits 1 \n", + "job unskilled resident \n", + "num_dependents 2 \n", + "own_telephone none \n", + "foreign_worker yes \n", + "class good \n", + "application_timestamp 2023-10-03 23:06:03 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check data (first few records, transposed for readability)\n", + "df.head(3).T" + ] + }, + { + "cell_type": "markdown", + "id": "72b2105a-b459-4715-aa53-6fe69fc4a210", + "metadata": {}, + "source": [ + "We'll also generate counterpart IDs and timestamps on the label data. In a real-life scenario, the label data would come separate and later relative to the loan application data. To mimic this, let's create a labels dataset with an \"outcome_timestamp\" column with a variable lag from the application timestamp of 30 to 90 days." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e214478b-ed9b-4354-ba6f-4117813c56c3", + "metadata": {}, + "outputs": [], + "source": [ + "# Add (lagged) label timestamps (30 to 90 days)\n", + "def lag_delta(data, seed):\n", + " np.random.seed(seed)\n", + " delta_days = np.random.randint(30, 90, len(data))\n", + " delta_hours = np.random.randint(0, 24, len(data))\n", + " delta = np.array([dt.timedelta(days=int(delta_days[i]), hours=int(delta_hours[i])) for i in range(len(data))])\n", + " return delta\n", + "\n", + "labels = df[[\"ID\", \"class\"]]\n", + "labels[\"outcome_timestamp\"] = pd.to_datetime(df.application_timestamp + lag_delta(df, SEED))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "356a7225-db20-4c15-87a3-4a0eb3127475", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check labels\n", + "labels.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "4a29f754-f758-402b-ac42-2dcfcee3b7fc", + "metadata": {}, + "source": [ + "You can verify that the `outcome timestamp` has a difference of 30 to 90 days from the \"application_timestamp\" (above)." + ] + }, + { + "cell_type": "markdown", + "id": "e720ce24-e092-4fcd-be3e-68bb18f4d2a7", + "metadata": {}, + "source": [ + "### Save Data" + ] + }, + { + "cell_type": "markdown", + "id": "5cae0578-8431-46c7-8d64-e52146f47d46", + "metadata": {}, + "source": [ + "Now that we have our data prepared, let's save it to local parquet files in the `data` directory (parquet is one of the file formats supported by Feast).\n", + "\n", + "One more step we will add is splitting the context data column-wise and saving it in two files. This step is contrived--we don't usually split data when we don't need to--but it will allow us to demonstrate later how Feast can easily join datasets (a common need in Data Science projects)." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "cebef56c-1f54-4d31-a545-75d708d38579", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the data directory if it doesn't exist\n", + "os.makedirs(\"Feature_Store/data\", exist_ok=True)\n", + "\n", + "# Split columns and save context data\n", + "a_cols = [\n", + " 'ID', 'checking_status', 'duration', 'credit_history', 'purpose',\n", + " 'credit_amount', 'savings_status', 'employment', 'application_timestamp',\n", + " 'installment_commitment', 'personal_status', 'other_parties',\n", + "]\n", + "b_cols = [\n", + " 'ID', 'residence_since', 'property_magnitude', 'age', 'other_payment_plans',\n", + " 'housing', 'existing_credits', 'job', 'num_dependents', 'own_telephone',\n", + " 'foreign_worker', 'application_timestamp'\n", + "]\n", + "\n", + "df[a_cols].to_parquet(\"Feature_Store/data/data_a.parquet\", engine=\"pyarrow\")\n", + "df[b_cols].to_parquet(\"Feature_Store/data/data_b.parquet\", engine=\"pyarrow\")\n", + "\n", + "# Save label data\n", + "labels.to_parquet(\"Feature_Store/data/labels.parquet\", engine=\"pyarrow\")" + ] + }, + { + "cell_type": "markdown", + "id": "d8d5de9f-bd27-4e95-802c-b121743dd1b0", + "metadata": {}, + "source": [ + "We have saved the following files to the `Feature_Store/data` directory: \n", + "- `data_a.parquet` (training data, a columns)\n", + "- `data_b.parquet` (training data, b columns)\n", + "- `labels.parquet` (label outcomes)" + ] + }, + { + "cell_type": "markdown", + "id": "af6355dc-ff5b-4b3f-b0bd-3c4020ef67e8", + "metadata": {}, + "source": [ + "With the feature data prepared, we are ready to setup and deploy the feature store. \n", + "\n", + "Continue with the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb new file mode 100644 index 00000000000..f736cdaed93 --- /dev/null +++ b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb @@ -0,0 +1,801 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "08d9e060-d455-43e2-b1ec-51e2a53e3169", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "93095241-3886-44a2-83b1-2a9537c21bc8", + "metadata": {}, + "source": [ + "# Deploying the Feature Store" + ] + }, + { + "cell_type": "markdown", + "id": "465783da-18eb-4945-98e7-bb1058a7af1b", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "11961d1b-72db-48dc-a07d-dcea9ba223b4", + "metadata": {}, + "source": [ + "Feast enables AI/ML teams to serve (and consume) features via feature stores. In this notebook, we will configure the feature stores and feature definitions, and deploy a Feast feature store server. We will also materialize (move) data from the offline store to the online store.\n", + "\n", + "In Feast, offline stores support pulling large amounts of data for model training using tools like Redshift, Snowflake, Bigquery, and Spark. In contrast, the focus of Feast online stores is feature serving in support of model inference, using tools like Redis, Snowflake, PostgreSQL, and SQLite.\n", + "\n", + "In this notebook, we will setup a file-based (Dask) offline store and SQLite online store. The online store will be made available through the Feast server." + ] + }, + { + "cell_type": "markdown", + "id": "dfed8ccf-0d7d-46a1-82f0-5765f8796088", + "metadata": {}, + "source": [ + "This notebook assumes that you have prepared the data by running the notebook [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb). " + ] + }, + { + "cell_type": "markdown", + "id": "e66b7a08-5d15-4804-a82a-8bc571777496", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "1c1e87a4-900b-48f3-a400-ce6608046ce3", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8bd21689-4a8e-4b0c-937d-0911df9db1d3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import re\n", + "import sys\n", + "import time\n", + "import signal\n", + "import sqlite3\n", + "import subprocess\n", + "import datetime as dt\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "markdown", + "id": "471db4b0-ea93-47a1-9d55-a80e4d2bdc1e", + "metadata": {}, + "source": [ + "### Feast Feature Store Configuration" + ] + }, + { + "cell_type": "markdown", + "id": "0a307490-4121-4bf3-a5c4-77a8885a4f6a", + "metadata": {}, + "source": [ + "For model training, we usually don't need (or want) a constantly running feature server. All we need is the ability to efficiently query and pull all of the training data at training time. In contrast, during model serving we need servers that are always ready to supply feature records in response to application requests. \n", + "\n", + "This training-serving dichotomy is reflected in Feast using \"offline\" and \"online\" stores. Offline stores are configured to work with database technologies typically used for training, while online stores are configured to use storage and streaming technologies that are popular for feature serving.\n", + "\n", + "We need to create a `feature_store.yaml` config file to tell feast the structure we want in our offline and online feature stores. Below, we write the configuration for a local \"Dask\" offline store and local SQLite online store. We give the feature store a project name of `loan_applications`, and provider `local`. The registry is where the feature store will keep track of feature definitions and online store updates; we choose a file location in this case.\n", + "\n", + "See the [feature_store.yaml](https://docs.feast.dev/reference/feature-repository/feature-store-yaml) documentation for further details. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b3757221-2037-49eb-867f-b9529fec06e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_store.yaml\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_store.yaml\n", + "\n", + "project: loan_applications\n", + "registry: data/registry.db\n", + "provider: local\n", + "offline_store:\n", + " type: dask\n", + "online_store:\n", + " type: sqlite\n", + " path: data/online_store.db\n", + "entity_key_serialization_version: 2" + ] + }, + { + "cell_type": "markdown", + "id": "180038f3-e5ce-4cce-bdf0-118eee7a822d", + "metadata": {}, + "source": [ + "### Feature Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "dd44b206-1f5c-4f55-bbab-41ba2d3f5202", + "metadata": {}, + "source": [ + "We also need to create feature definitions and other feature constructs in a python file, which we name `feature_definitions.py`. For our purposes, we define the following:\n", + "\n", + "- Data Source: connections to data storage or data-producing endpoints\n", + "- Entity: primary key fields which can be used for joining data\n", + "- FeatureView: collections of features from a data source\n", + "\n", + "For more information on these, see the [Concepts](https://docs.feast.dev/getting-started/concepts) section of the Feast documentation." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d3e8fd80-0bee-463c-b3fb-bd0d1ee83a9c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_definitions.py\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_definitions.py\n", + "\n", + "# Imports\n", + "import os\n", + "from pathlib import Path\n", + "from feast import (\n", + " FileSource,\n", + " Entity,\n", + " FeatureView,\n", + " Field,\n", + " FeatureService\n", + ")\n", + "from feast.types import Float32, String\n", + "from feast.data_format import ParquetFormat\n", + "\n", + "CURRENT_DIR = os.path.abspath(os.curdir)\n", + "\n", + "# Data Sources\n", + "# A data source tells Feast where the data lives\n", + "data_a = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_a.parquet\").as_uri()\n", + ")\n", + "data_b = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_b.parquet\").as_uri()\n", + ")\n", + "\n", + "# Entity\n", + "# An entity tells Feast the column it can use to join tables\n", + "loan_id = Entity(\n", + " name = \"loan_id\",\n", + " join_keys = [\"ID\"]\n", + ")\n", + "\n", + "# Feature views\n", + "# A feature view is how Feast groups features\n", + "features_a = FeatureView(\n", + " name=\"data_a\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"checking_status\", dtype=String),\n", + " Field(name=\"duration\", dtype=Float32),\n", + " Field(name=\"credit_history\", dtype=String),\n", + " Field(name=\"purpose\", dtype=String),\n", + " Field(name=\"credit_amount\", dtype=Float32),\n", + " Field(name=\"savings_status\", dtype=String),\n", + " Field(name=\"employment\", dtype=String),\n", + " Field(name=\"installment_commitment\", dtype=Float32),\n", + " Field(name=\"personal_status\", dtype=String),\n", + " Field(name=\"other_parties\", dtype=String),\n", + " ],\n", + " source=data_a\n", + ")\n", + "features_b = FeatureView(\n", + " name=\"data_b\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"residence_since\", dtype=Float32),\n", + " Field(name=\"property_magnitude\", dtype=String),\n", + " Field(name=\"age\", dtype=Float32),\n", + " Field(name=\"other_payment_plans\", dtype=String),\n", + " Field(name=\"housing\", dtype=String),\n", + " Field(name=\"existing_credits\", dtype=Float32),\n", + " Field(name=\"job\", dtype=String),\n", + " Field(name=\"num_dependents\", dtype=Float32),\n", + " Field(name=\"own_telephone\", dtype=String),\n", + " Field(name=\"foreign_worker\", dtype=String),\n", + " ],\n", + " source=data_b\n", + ")\n", + "\n", + "# Feature Service\n", + "# a feature service in Feast represents a logical group of features\n", + "loan_fs = FeatureService(\n", + " name=\"loan_fs\",\n", + " features=[features_a, features_b]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b47c1b5-849e-43f3-8043-60466aaed69f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "be9723eb-8fa0-4338-b50c-f9f1ff6bb13a", + "metadata": {}, + "source": [ + "### Applying the Configuration and Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "c796d45f-28c0-4875-bbb1-71e5a15dcb96", + "metadata": {}, + "source": [ + "Now that we have our feature store configuration (`feature_store.yaml`) and feature definitions (`feature_definitions.py`), we are ready to \"apply\" them. The `feast apply` command creates a registry file (`Feature_Store/data/registry.db`) and sets up data connections; in this case, it creates a SQLite database (`Feature_Store/data/online_store.db`)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "394467f3-4ced-492a-9379-105aea9d4a6d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created entity \u001b[1m\u001b[32mloan_id\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_a\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_b\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mloan_fs\u001b[0m\n", + "\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_a\u001b[0m\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_b\u001b[0m\n", + "\n" + ] + } + ], + "source": [ + "# Run 'feast apply' in the Feature_Store directory\n", + "!feast --chdir ./Feature_Store apply" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e32f40eb-a31a-4877-8f40-2d8515302f39", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 232\n", + "-rw-r--r-- 1 501 20 33K Oct 27 14:17 data_a.parquet\n", + "-rw-r--r-- 1 501 20 27K Oct 27 14:17 data_b.parquet\n", + "-rw-r--r-- 1 501 20 17K Oct 27 14:17 labels.parquet\n", + "-rw-r--r-- 1 501 20 28K Oct 27 14:19 online_store.db\n", + "-rw-r--r-- 1 501 20 2.8K Oct 27 14:19 registry.db\n" + ] + } + ], + "source": [ + "# List the Feature_Store/data/ directory to see newly created files\n", + "!ls -nlh Feature_Store/data/" + ] + }, + { + "cell_type": "markdown", + "id": "31014885-ce6a-4007-8bdb-d74d3b44781b", + "metadata": {}, + "source": [ + "Note that while `feast apply` set up the `sqlite` online database, `online_store.db`, no data has been added to the online database as of yet. We can verify this by connecting with the `sqlite3` library." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "107ca856-af06-40c4-8339-70daf59cdf37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online Store Tables: [('loan_applications_data_a',), ('loan_applications_data_b',)]\n", + "loan_applications_data_a data: []\n", + "loan_applications_data_b data: []\n" + ] + } + ], + "source": [ + "# Connect to sqlite database\n", + "conn = sqlite3.connect(\"Feature_Store/data/online_store.db\")\n", + "cursor = conn.cursor()\n", + "# Query table data (3 tables)\n", + "print(\n", + " \"Online Store Tables: \",\n", + " cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table';\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_a data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_a\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_b data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_b\").fetchall()\n", + ")\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "id": "03b927ee-7913-4a8a-b17b-9bee361d8d94", + "metadata": {}, + "source": [ + "Since we have used `feast apply` to create the registry, we can now use the Feast Python SDK to interact with our new feature store. To see other possible commands see the [Feast Python SDK documentation](https://rtd.feast.dev/en/master/)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c764a60a-b911-41a8-ba8f-7ef0a0bc7257", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RepoConfig(project='loan_applications', provider='local', registry_config='data/registry.db', online_config={'type': 'sqlite', 'path': 'data/online_store.db'}, offline_config={'type': 'dask'}, batch_engine_config='local', feature_server=None, flags=None, repo_path=PosixPath('Feature_Store'), entity_key_serialization_version=2, coerce_tz_aware=True)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get feature store config\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "store.config" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fc572976-6ce9-44f6-8b67-28ee6157e29c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature view: data_a | Features: [checking_status-String, duration-Float32, credit_history-String, purpose-String, credit_amount-Float32, savings_status-String, employment-String, installment_commitment-Float32, personal_status-String, other_parties-String]\n", + "Feature view: data_b | Features: [residence_since-Float32, property_magnitude-String, age-Float32, other_payment_plans-String, housing-String, existing_credits-Float32, job-String, num_dependents-Float32, own_telephone-String, foreign_worker-String]\n" + ] + } + ], + "source": [ + "# List feature views\n", + "feature_views = store.list_batch_feature_views()\n", + "for fv in feature_views:\n", + " print(f\"Feature view: {fv.name} | Features: {fv.features}\")" + ] + }, + { + "cell_type": "markdown", + "id": "027edcfe-58d7-4dcb-92e2-5a5514c0f1f0", + "metadata": {}, + "source": [ + "### Deploying the Feature Store Servers" + ] + }, + { + "cell_type": "markdown", + "id": "c9aab68d-395f-421e-ba11-ad8c4acc9d6f", + "metadata": {}, + "source": [ + "If you wish to share a feature store with your team, Feast provides feature servers. To spin up an offline feature server process, we can use the `feast serve_offline` command, while to spin up a Feast online feature server, we use the `feast serve` command.\n", + "\n", + "Let's spin up an offline and an online server that we can use in the subsequent notebooks to get features during model training and model serving. We will run both servers as background processes, that we can communicate with in the other notebooks.\n", + "\n", + "First, we write a helper function to extract the first few printed log lines (so we can print it in the notebook cell output)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "568f81b8-df34-4b06-8a3f-1a6bdc2e6cff", + "metadata": {}, + "outputs": [], + "source": [ + "# TimeoutError class\n", + "class TimeoutError(Exception):\n", + " pass\n", + "\n", + "# TimeoutError raise function\n", + "def timeout():\n", + " raise TimeoutError(\"timeout\")\n", + "\n", + "# Get first few log lines function\n", + "def print_first_proc_lines(proc, wait):\n", + " '''Given a process, `proc`, read and print output lines until they stop \n", + " comming (waiting up to `wait` seconds for new lines to appear)'''\n", + " lines = \"\"\n", + " while True:\n", + " signal.signal(signal.SIGALRM, timeout)\n", + " signal.alarm(wait)\n", + " try:\n", + " lines += proc.stderr.readline()\n", + " except:\n", + " break\n", + " if lines:\n", + " print(lines, file=sys.stderr)" + ] + }, + { + "cell_type": "markdown", + "id": "88d25a87-241a-46c6-9ca7-d035959c5f74", + "metadata": {}, + "source": [ + "Launch the offline server with the command `feast --chdir ./Feature_Store serve_offline`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ce965dd4-652b-4c36-a064-fd0fd97d3ef7", + "metadata": {}, + "outputs": [], + "source": [ + "# Feast offline server process\n", + "offline_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve_offline 2>&2 & echo $! > server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(offline_server_proc, 2)" + ] + }, + { + "cell_type": "markdown", + "id": "59958d64-8e68-45ff-9549-556cbf46908c", + "metadata": {}, + "source": [ + "The tail end of the command above, `2>&2 & echo $! > server_proc.txt`, captures log messages (in the offline case there are none), and writes the process PID to the file `server_proc.txt` (we will use this in the cleanup notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb))." + ] + }, + { + "cell_type": "markdown", + "id": "cfed4334-9e62-4f3f-be96-3f7db2f06ada", + "metadata": {}, + "source": [ + "Next, launch the online server with the command `feast --chdir ./Feature_Store serve`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a581fbe2-13ba-433e-8e76-dc82cc22af74", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ddowler/Code/Feast/feast/examples/credit-risk-end-to-end/venv-py3.11/lib/python3.11/site-packages/uvicorn/workers.py:16: DeprecationWarning: The `uvicorn.workers` module is deprecated. Please use `uvicorn-worker` package instead.\n", + "For more details, see https://github.com/Kludex/uvicorn-worker.\n", + " warnings.warn(\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Starting gunicorn 23.0.0\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Listening at: http://127.0.0.1:6566 (44621)\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Using worker: uvicorn.workers.UvicornWorker\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Booting worker with pid: 44623\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Started server process [44623]\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Waiting for application startup.\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Application startup complete.\n", + "\n" + ] + } + ], + "source": [ + "# Feast online server (master and worker) processes\n", + "online_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve 2>&2 & echo $! >> server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(online_server_proc, 3)" + ] + }, + { + "cell_type": "markdown", + "id": "0e778173-f58a-4074-b63f-107e1f39577b", + "metadata": {}, + "source": [ + "Note that the output helpfully let's us know that the online server is \"Listening at: http://127.0.0.1:6566\" (the default host:port).\n", + "\n", + "List the running processes to verify they are up." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9b1a224d-884d-45c5-9711-2e2eb4351710", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 44594 1 0 2:19PM ?? 0:03.66 **/python **/feast --chdir ./Feature_Store serve_offline\n", + " 501 44621 1 0 2:19PM ?? 0:03.58 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44623 44621 0 2:19PM ?? 0:00.03 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44662 44542 0 2:19PM ?? 0:00.01 /bin/zsh -c ps -ef | grep **/feast | grep serve\n" + ] + } + ], + "source": [ + "# List running Feast processes (paths redacted)\n", + "running_procs = !ps -ef | grep feast | grep serve\n", + "\n", + "for line in running_procs:\n", + " redacted = re.sub(r'/*[^\\s]*(?P(python )|(feast ))', r'**/\\g', line)\n", + " print(redacted)" + ] + }, + { + "cell_type": "markdown", + "id": "fd52eeb4-948c-472b-9111-8549fda955a1", + "metadata": {}, + "source": [ + "Note that there are two process for the online server (master and worker)." + ] + }, + { + "cell_type": "markdown", + "id": "8258e7a8-5f6e-4737-93ee-63591518b169", + "metadata": {}, + "source": [ + "### Materialize Features to the Online Store" + ] + }, + { + "cell_type": "markdown", + "id": "21b354ab-ec22-476d-8fd9-6ffe0f3fbacb", + "metadata": {}, + "source": [ + "At this point, there is no data in the online store yet. Let's use the SDK feature store object (that we created above) to \"materialize\" data; this is Feast lingo for moving/updating data from the offline store to the online store." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ff6146df-03a7-4ac2-a665-ee5f440c3605", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views from \u001b[1m\u001b[32m2023-09-24 12:00:00-06:00\u001b[0m to \u001b[1m\u001b[32m2024-01-07 12:00:00-07:00\u001b[0m into the \u001b[1m\u001b[32msqlite\u001b[0m online store.\n", + "\n", + "\u001b[1m\u001b[32mdata_a\u001b[0m:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1000 [00:00=7\",\"4<=X<7\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"existing paid\",\"critical/other existing credit\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"female div/dep/mar\",\"male mar/wid\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[12579.0,2463.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"used car\",\"new car\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[24.0,24.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[4.0,4.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[2.0,3.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,1.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[44.0,27.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"for free\",\"own\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,2.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"high qualif/self emp/mgmt\",\"skilled\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"no known property\",\"life insurance\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]}]}']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response" + ] + }, + { + "cell_type": "markdown", + "id": "01d20196-1d42-486d-a0bd-97193c953785", + "metadata": {}, + "source": [ + "The `curl` command gave us a quick validation. In the [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) notebook, we'll use the Python `requests` library to handle the query better." + ] + }, + { + "cell_type": "markdown", + "id": "d74a5117-dd34-4dde-93a8-ea6e8c4c545a", + "metadata": {}, + "source": [ + "Now that the feature stores and their respective servers have been configured and deployed, we can proceed to train an AI model in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb new file mode 100644 index 00000000000..ca0d0e29d95 --- /dev/null +++ b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb @@ -0,0 +1,1541 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "54f2ab19-68e1-4725-b6e7-efd8eedebe1a", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "69a40de4-65cf-4b45-b321-2b7ce571f8cb", + "metadata": {}, + "source": [ + "# Credit Risk Model Training" + ] + }, + { + "cell_type": "markdown", + "id": "fe641d83-1e28-4f7f-895c-8ca038f6cc53", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "8f04f635-401b-47b6-b807-df61d42ec752", + "metadata": {}, + "source": [ + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application process). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we train our AI model. We will use the popular scikit-learn library (sklearn) to train a RandomForestClassifier, as this is a relatively easy choice for a baseline model." + ] + }, + { + "cell_type": "markdown", + "id": "a96bf1aa-c450-4201-83a4-e25b08bdd12d", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "a47b33bc-bc06-4de0-8f3a-beea8179035c", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c66a3dab-fdbf-40be-8227-6180dc314a84", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import warnings\n", + "import datetime\n", + "import feast\n", + "import joblib\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", + "from feast import FeatureStore, RepoConfig\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OrdinalEncoder\n", + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import classification_report" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a841445-fa47-4826-a874-28ac0e4ea57f", + "metadata": {}, + "outputs": [], + "source": [ + "# Ignore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "23579727-7797-4101-a70d-b0d4c24b0fdf", + "metadata": {}, + "outputs": [], + "source": [ + "# Random seed\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "fc5be519-7733-449b-8dc3-411e86371315", + "metadata": {}, + "source": [ + "This notebook assumes that you have already done the following:\n", + "\n", + "1. Run the [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) notebook to prepare the data.\n", + "2. Run the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook to configure the feature stores and launch the feature store servers.\n", + "\n", + "If you have not completed the above steps, please go back and do so before continuing. This notebook relies on the data prepared by 1, and it uses the Feast offline server stood up by 2." + ] + }, + { + "cell_type": "markdown", + "id": "1ca99047-e508-4b1f-9f4c-f11e38587d70", + "metadata": {}, + "source": [ + "### Load Label (Outcome) Data" + ] + }, + { + "cell_type": "markdown", + "id": "89b49268-b7a5-4abc-8d82-1cdbf9bb4473", + "metadata": {}, + "source": [ + "From our previous data exploration, remember that the label data represents whether the loan was classed as \"good\" (1) or \"bad\" (0). Let's pull the labels for training, as we will use them as our \"entity dataframe\" when pulling features.\n", + "\n", + "This is also a good time to remember that the label timestamps are lagged by 30-90 days from the context data records." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6a227a12-7b3e-462a-8f6e-38a7690df1c4", + "metadata": {}, + "outputs": [], + "source": [ + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "31a39cad-0a85-4d98-ad95-008c81bb6fe0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
33good2023-11-17 07:37:19
44bad2023-12-01 05:01:48
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03\n", + "3 3 good 2023-11-17 07:37:19\n", + "4 4 bad 2023-12-01 05:01:48" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels.head()" + ] + }, + { + "cell_type": "markdown", + "id": "857f29fd-46d3-444b-b24f-eaccd82ab7d3", + "metadata": {}, + "source": [ + "### Pull Feature Data from Feast Offline Store" + ] + }, + { + "cell_type": "markdown", + "id": "07c13b69-3d26-484c-97cd-97734cc812bd", + "metadata": {}, + "source": [ + "In order to pull feature data from the offline store, we create a FeatureStore object that connects to the offline server (continuously running in the previous notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9e9828f8-f210-4586-ac36-3f7e17f4f1e8", + "metadata": {}, + "outputs": [], + "source": [ + "# Create FeatureStore object\n", + "# (connects to the offline server deployed in 02_Deploying_the_Feature_Store.ipynb) \n", + "store = FeatureStore(config=RepoConfig(\n", + " project=\"loan_applications\",\n", + " provider=\"local\",\n", + " registry=\"Feature_Store/data/registry.db\",\n", + " offline_store={\n", + " \"type\": \"remote\",\n", + " \"host\": \"localhost\",\n", + " \"port\": 8815\n", + " },\n", + " entity_key_serialization_version=2\n", + "))" + ] + }, + { + "cell_type": "markdown", + "id": "c007e7ca-40c1-4850-abed-73b6171ad08d", + "metadata": {}, + "source": [ + "Now, we can retrieve feature data by supplying our entity dataframe and feature specifications to the `get_historical_features` function. Note that this function performs a fuzzy lookback (\"point-in-time\") join, matching the lagged outcome timestamp to the closest application timestamp (per ID) in the context data; it also joins the \"a\" and \"b\" features that we had previously split into two tables.\n", + "\n", + "To keep this example simple, we will limit our feature set to the numerical features plus two categorical features." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dd2e3cb5-c865-48f4-80b6-8a14a1ff09ab", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using outcome_timestamp as the event timestamp. To specify a column explicitly, please name it event_timestamp.\n" + ] + } + ], + "source": [ + "# Get feature data\n", + "# (Joins a and b data, and selects records with the right timestamps)\n", + "df = store.get_historical_features(\n", + " entity_df=labels,\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c72f6cb1-bbbf-4512-98cd-0abe5ff0c24b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 12 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 ID 1000 non-null int64 \n", + " 1 class 1000 non-null category \n", + " 2 outcome_timestamp 1000 non-null datetime64[ns, UTC]\n", + " 3 duration 1000 non-null int64 \n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 installment_commitment 1000 non-null int64 \n", + " 6 checking_status 1000 non-null category \n", + " 7 residence_since 1000 non-null int64 \n", + " 8 age 1000 non-null int64 \n", + " 9 existing_credits 1000 non-null int64 \n", + " 10 num_dependents 1000 non-null int64 \n", + " 11 housing 1000 non-null category \n", + "dtypes: category(3), datetime64[ns, UTC](1), int64(8)\n", + "memory usage: 73.8 KB\n" + ] + } + ], + "source": [ + "# Check the data info\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "110ea48c-0a5a-4642-aaba-a9eeb4a7da48", + "metadata": {}, + "source": [ + "### Split the Data" + ] + }, + { + "cell_type": "markdown", + "id": "f6669dce-a8b0-4d80-9a15-70b7dfd2d718", + "metadata": {}, + "source": [ + "Next, we split the data into a `train` and `validate` set, which we will use to train and then validate a model. The validation set will allow us to more accurately assess the model's performance on data that it has not seen during the training phase." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "036b0a54-48e4-4414-bb8c-0c30b6ab7469", + "metadata": {}, + "outputs": [], + "source": [ + "# Split data into train and validate datasets\n", + "train, validate = train_test_split(df, test_size=0.2, random_state=SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "4b65cbf7-5981-4f51-97aa-a3ff7027f2f3", + "metadata": {}, + "source": [ + "### Exploratory Data Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "e516ded8-10ad-4274-a736-f288290b5883", + "metadata": {}, + "source": [ + "Before building a model, a data scientist needs to gain understanding of the data to make sure it meets important statistical assumptions, and to identify potential opportunities and issues. As the purpose of this particular example is to show working with Feast, we will take the view of a data scientist looking to build a quick baseline model to establish some low-end metrics.\n", + "\n", + "Note that this data set is very \"clean\", as it has already been prepared. In real-life, production credit risk data can be much more complex, and have many issues that need to be understood and addressed before modeling." + ] + }, + { + "cell_type": "markdown", + "id": "553986a0-c804-4ab4-a4b9-48b16c72fd4f", + "metadata": {}, + "source": [ + "Let's look at counts for the target variable `class`, which tells us whether a (historical) loan was good or bad. We can see that there were many more good loans than bad, making the dataset imbalanced." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "607bd29b-eaf4-41a6-aaca-a8eaaf37e2d2", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAysElEQVR4nO3deVxV5d7///dmRmWDoIKWqJWKOHZw2k1qkWRkeWvllKlHG8FKy2PcOWLedqwcQ6tTqWWm2WBq5kRZHcVSTFNT1MqwFCgVtnoUBNbvj37sb/ugpQhsvHw9H4/1yHVd11rrc+3d1jdr2Ngsy7IEAABgKC9PFwAAAFCRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIO8AlyGazafz48Z4u45LD6wZcngg7QDmYN2+ebDab21KnTh116dJFn3zyiafLq3T5+fmaNWuWbrjhBtWsWVN+fn6qV6+e7rzzTr3zzjsqKirydInndODAAdlsNr3wwgueLuWCOZ1OTZgwQa1bt1aNGjUUGBioFi1aaNSoUTp06JCny5MkrVy5ksCJSufj6QIAkyQnJ6tRo0ayLEvZ2dmaN2+ebr/9di1fvlx33HGHp8urFL/++qu6deum9PR0xcXFafTo0QoNDVVWVpbWrVunfv36af/+/RozZoynSzXKDz/8oNjYWGVmZuqee+7Rgw8+KD8/P3377bd6/fXX9eGHH2rv3r2eLlMrV65USkoKgQeVirADlKNu3bqpbdu2rvUhQ4YoPDxc77zzzmUTdgYMGKBvvvlG77//vnr27OnWl5SUpC1btigjI8ND1ZmpsLBQPXv2VHZ2ttavX68bbrjBrX/SpEn65z//6aHqAM/jMhZQgUJCQhQYGCgfH/efK1544QVdd911CgsLU2BgoGJiYvTee++V2j4/P1/Dhw9X7dq1FRQUpDvvvFM///zzXx43OztbPj4+mjBhQqm+jIwM2Ww2vfTSS5KkM2fOaMKECWrcuLECAgIUFhamG264QWvXrr3g+aalpWn16tV68MEHSwWdEm3btlX//v3d2nJyclzBMCAgQK1bt9b8+fNLbXvy5Ek9+eSTql+/vvz9/dW0aVO98MILsizLbVxZX7cLcb41n+97bbPZlJiYqKVLl6pFixby9/dX8+bNtWrVqr+s5f3339f27dv1zDPPlAo6kmS32zVp0iS3tiVLligmJkaBgYGqVauW7rvvPv3yyy9uYzp37qzOnTuX2t+gQYPUsGFD1/ofL/29+uqruvrqq+Xv76927dpp8+bNbtulpKS45luyABWNMztAOcrLy9Nvv/0my7KUk5OjWbNm6cSJE7rvvvvcxs2YMUN33nmn+vfvr4KCAi1atEj33HOPVqxYofj4eNe4oUOHasGCBerXr5+uu+46ffrpp2795xIeHq5OnTrp3Xff1bhx49z6Fi9eLG9vb91zzz2SpPHjx2vy5MkaOnSo2rdvL6fTqS1btmjr1q269dZbL2j+y5cvl6RS8/0zp06dUufOnbV//34lJiaqUaNGWrJkiQYNGqTc3Fw9/vjjkiTLsnTnnXfqs88+05AhQ9SmTRutXr1aI0eO1C+//KJp06a59lnW1628a5bO/72WpH//+9/64IMP9OijjyooKEgzZ85Ur169lJmZqbCwsHPWs2zZMkm/n1U7H/PmzdPgwYPVrl07TZ48WdnZ2ZoxY4Y2bNigb775RiEhIRf+okhauHChjh8/roceekg2m01TpkxRz5499cMPP8jX11cPPfSQDh06pLVr1+qtt94q0zGAMrEAXLS5c+dakkot/v7+1rx580qN/89//uO2XlBQYLVo0cK6+eabXW3btm2zJFmPPvqo29h+/fpZkqxx48b9aU2vvPKKJcnasWOHW3t0dLTbcVq3bm3Fx8ef71T/1P/8z/9Ykqzc3Fy39lOnTlm//vqrazl27Jirb/r06ZYka8GCBa62goICy+FwWDVq1LCcTqdlWZa1dOlSS5L17LPPuu377rvvtmw2m7V//37Lsi7+dfvxxx8tSdbzzz9/zjHnW7Nlnd97bVmWJcny8/NzzcOyLGv79u2WJGvWrFl/WvO1115rBQcH/+mYPx6/Tp06VosWLaxTp0652lesWGFJssaOHetq69Spk9WpU6dS+xg4cKDVoEED13rJaxYWFmYdPXrU1f7RRx9Zkqzly5e72hISEiz+6UFl4zIWUI5SUlK0du1arV27VgsWLFCXLl00dOhQffDBB27jAgMDXX8+duyY8vLydOONN2rr1q2u9pUrV0qSHnvsMbdtn3jiifOqpWfPnvLx8dHixYtdbTt37tR3332n3r17u9pCQkK0a9cu7du377zneS5Op1OSVKNGDbf2l19+WbVr13Ytf7zUsnLlSkVERKhv376uNl9fXz322GM6ceKEPv/8c9c4b2/vUq/Hk08+KcuyXE+9Xezrdj7Ot2bp/N7rErGxsbr66qtd661atZLdbtcPP/zwp/U4nU4FBQWdV+1btmxRTk6OHn30UQUEBLja4+PjFRUVpY8//vi89nM2vXv3Vs2aNV3rN954oyT9Zf1ARSPsAOWoffv2io2NVWxsrPr376+PP/5Y0dHRSkxMVEFBgWvcihUr1LFjRwUEBCg0NFS1a9fWnDlzlJeX5xrz008/ycvLy+0fP0lq2rTpedVSq1Yt3XLLLXr33XddbYsXL5aPj4/b/TTJycnKzc1VkyZN1LJlS40cOVLffvttmeZf8g/uiRMn3Np79erlCoGtWrVy6/vpp5/UuHFjeXm5/3XUrFkzV3/Jf+vVq1fqH/WzjbuY1+18nG/N0vm91yUiIyNLtdWsWVPHjh3703rsdruOHz9+3rVLZ389oqKi3Gq/UP9df0nw+av6gYpG2AEqkJeXl7p06aLDhw+7zpx8+eWXuvPOOxUQEKDZs2dr5cqVWrt2rfr161fqRtuL1adPH+3du1fbtm2TJL377ru65ZZbVKtWLdeYm266Sd9//73eeOMNtWjRQq+99pr+9re/6bXXXrvg40VFRUn6/QzSH9WvX98VAv/4k7/pLvS99vb2Put+/ur/i6ioKOXl5engwYPlUneJc908fK7vSSpr/UBFI+wAFaywsFDS/zvb8f777ysgIECrV6/W3//+d3Xr1k2xsbGltmvQoIGKi4v1/fffu7VfyGPbPXr0kJ+fnxYvXqxt27Zp79696tOnT6lxoaGhGjx4sN555x0dPHhQrVq1KtP3oJQ8Xv/222+f9zYNGjTQvn37VFxc7Na+Z88eV3/Jfw8dOlTqDMbZxl3s61ZeNZ/ve32xunfvLklasGDBX44tqe1sr0dGRoarX/r9zExubm6pcRdz9oenr+AJhB2gAp05c0Zr1qyRn5+f6xKHt7e3bDab20/HBw4c0NKlS9227datmyRp5syZbu3Tp08/7+OHhIQoLi5O7777rhYtWiQ/Pz/16NHDbcyRI0fc1mvUqKFrrrlG+fn5rra8vDzt2bPnrJde/uj666/XrbfeqldffVUfffTRWcf890/5t99+u7KystzuLSosLNSsWbNUo0YNderUyTWuqKjI9ch8iWnTpslms7ler/J43f7K+dZ8vu/1xbr77rvVsmVLTZo0SWlpaaX6jx8/rmeeeUbS74/+16lTRy+//LLbe/zJJ59o9+7dbk+IXX311dqzZ49+/fVXV9v27du1YcOGMtdavXp1STpriAIqCo+eA+Xok08+cf10n5OTo4ULF2rfvn16+umnZbfbJf1+I+jUqVN12223qV+/fsrJyVFKSoquueYat3tl2rRpo759+2r27NnKy8vTddddp9TUVO3fv/+Caurdu7fuu+8+zZ49W3FxcaUeK46Ojlbnzp0VExOj0NBQbdmyRe+9954SExNdYz788EMNHjxYc+fO1aBBg/70eAsWLNBtt92mHj16uM5k1KxZ0/UNyl988YUrkEjSgw8+qFdeeUWDBg1Senq6GjZsqPfee08bNmzQ9OnTXffodO/eXV26dNEzzzyjAwcOqHXr1lqzZo0++ugjPfHEE657dMrrdUtNTdXp06dLtffo0eO8az7f9/pi+fr66oMPPlBsbKxuuukm3Xvvvbr++uvl6+urXbt2aeHChapZs6YmTZokX19f/fOf/9TgwYPVqVMn9e3b1/XoecOGDTV8+HDXfv/+979r6tSpiouL05AhQ5STk6OXX35ZzZs3d92MfqFiYmIk/X4DeVxcnLy9vc96thEoV558FAwwxdkePQ8ICLDatGljzZkzxyouLnYb//rrr1uNGze2/P39raioKGvu3LnWuHHjSj2Se+rUKeuxxx6zwsLCrOrVq1vdu3e3Dh48eF6PUJdwOp1WYGBgqUelSzz77LNW+/btrZCQECswMNCKioqyJk2aZBUUFJSa39y5c8/rmKdOnbKmT59uORwOy263Wz4+PlZERIR1xx13WG+//bZVWFjoNj47O9saPHiwVatWLcvPz89q2bLlWY91/Phxa/jw4Va9evUsX19fq3Hjxtbzzz9f6vW9mNet5DHqcy1vvfXWBdV8vu+1JCshIaHU9g0aNLAGDhz4pzWXOHbsmDV27FirZcuWVrVq1ayAgACrRYsWVlJSknX48GG3sYsXL7auvfZay9/f3woNDbX69+9v/fzzz6X2uWDBAuuqq66y/Pz8rDZt2lirV68+56PnZ3tc/79f88LCQmvYsGFW7dq1LZvNxmPoqBQ2y+LOMQAAYC7u2QEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphR79/o6vT6eT3twAAYCDCjn7/KvXg4ODz/q3BAADg0kHYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNF8PF0AAFSW02eKlHn0P54uA7gsRIZWU4Cvt6fLkETYAXAZyTz6H439aKenywAuC8l3tVCT8CBPlyGJy1gAAMBwhB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDSPhp3x48fLZrO5LVFRUa7+06dPKyEhQWFhYapRo4Z69eql7Oxst31kZmYqPj5e1apVU506dTRy5EgVFhZW9lQAAEAV5ePpApo3b65169a51n18/l9Jw4cP18cff6wlS5YoODhYiYmJ6tmzpzZs2CBJKioqUnx8vCIiIrRx40YdPnxY999/v3x9ffV///d/lT4XAABQ9Xg87Pj4+CgiIqJUe15enl5//XUtXLhQN998syRp7ty5atasmTZt2qSOHTtqzZo1+u6777Ru3TqFh4erTZs2mjhxokaNGqXx48fLz8+vsqcDAACqGI/fs7Nv3z7Vq1dPV111lfr376/MzExJUnp6us6cOaPY2FjX2KioKEVGRiotLU2SlJaWppYtWyo8PNw1Ji4uTk6nU7t27TrnMfPz8+V0Ot0WAABgJo+GnQ4dOmjevHlatWqV5syZox9//FE33nijjh8/rqysLPn5+SkkJMRtm/DwcGVlZUmSsrKy3IJOSX9J37lMnjxZwcHBrqV+/frlOzEAAFBlePQyVrdu3Vx/btWqlTp06KAGDRro3XffVWBgYIUdNykpSSNGjHCtO51OAg8AAIby+GWsPwoJCVGTJk20f/9+RUREqKCgQLm5uW5jsrOzXff4RERElHo6q2T9bPcBlfD395fdbndbAACAmapU2Dlx4oS+//571a1bVzExMfL19VVqaqqrPyMjQ5mZmXI4HJIkh8OhHTt2KCcnxzVm7dq1stvtio6OrvT6AQBA1ePRy1hPPfWUunfvrgYNGujQoUMaN26cvL291bdvXwUHB2vIkCEaMWKEQkNDZbfbNWzYMDkcDnXs2FGS1LVrV0VHR2vAgAGaMmWKsrKyNHr0aCUkJMjf39+TUwMAAFWER8POzz//rL59++rIkSOqXbu2brjhBm3atEm1a9eWJE2bNk1eXl7q1auX8vPzFRcXp9mzZ7u29/b21ooVK/TII4/I4XCoevXqGjhwoJKTkz01JQAAUMXYLMuyPF2EpzmdTgUHBysvL4/7dwCD7c0+rrEf7fR0GcBlIfmuFmoSHuTpMiRVsXt2AAAAyhthBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjFZlws5zzz0nm82mJ554wtV2+vRpJSQkKCwsTDVq1FCvXr2UnZ3ttl1mZqbi4+NVrVo11alTRyNHjlRhYWElVw8AAKqqKhF2Nm/erFdeeUWtWrVyax8+fLiWL1+uJUuW6PPPP9ehQ4fUs2dPV39RUZHi4+NVUFCgjRs3av78+Zo3b57Gjh1b2VMAAABVlMfDzokTJ9S/f3/961//Us2aNV3teXl5ev311zV16lTdfPPNiomJ0dy5c7Vx40Zt2rRJkrRmzRp99913WrBggdq0aaNu3bpp4sSJSklJUUFBgaemBAAAqhCPh52EhATFx8crNjbWrT09PV1nzpxxa4+KilJkZKTS0tIkSWlpaWrZsqXCw8NdY+Li4uR0OrVr165zHjM/P19Op9NtAQAAZvLx5MEXLVqkrVu3avPmzaX6srKy5Ofnp5CQELf28PBwZWVlucb8MeiU9Jf0ncvkyZM1YcKEi6weAABcCjx2ZufgwYN6/PHH9fbbbysgIKBSj52UlKS8vDzXcvDgwUo9PgAAqDweCzvp6enKycnR3/72N/n4+MjHx0eff/65Zs6cKR8fH4WHh6ugoEC5ublu22VnZysiIkKSFBERUerprJL1kjFn4+/vL7vd7rYAAAAzeSzs3HLLLdqxY4e2bdvmWtq2bav+/fu7/uzr66vU1FTXNhkZGcrMzJTD4ZAkORwO7dixQzk5Oa4xa9euld1uV3R0dKXPCQAAVD0eu2cnKChILVq0cGurXr26wsLCXO1DhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6VPicAAFD1ePQG5b8ybdo0eXl5qVevXsrPz1dcXJxmz57t6vf29taKFSv0yCOPyOFwqHr16ho4cKCSk5M9WDUAAKhKbJZlWZ4uwtOcTqeCg4OVl5fH/TuAwfZmH9fYj3Z6ugzgspB8Vws1CQ/ydBmSqsD37AAAAFQkwg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAo5Up7Fx11VU6cuRIqfbc3FxdddVVF10UAABAeSlT2Dlw4ICKiopKtefn5+uXX3656KIAAADKi8+FDF62bJnrz6tXr1ZwcLBrvaioSKmpqWrYsGG5FQcAAHCxLijs9OjRQ5Jks9k0cOBAtz5fX181bNhQL774YrkVBwAAcLEuKOwUFxdLkho1aqTNmzerVq1aFVIUAABAebmgsFPixx9/LO86AAAAKkSZwo4kpaamKjU1VTk5Oa4zPiXeeOONiy4MAACgPJQp7EyYMEHJyclq27at6tatK5vNVt51AQAAlIsyhZ2XX35Z8+bN04ABA8q7HgAAgHJVpu/ZKSgo0HXXXVfetQAAAJS7MoWdoUOHauHCheVdCwAAQLkr02Ws06dP69VXX9W6devUqlUr+fr6uvVPnTq1XIoDAAC4WGUKO99++63atGkjSdq5c6dbHzcrAwCAqqRMl7E+++yzcy6ffvrpee9nzpw5atWqlex2u+x2uxwOhz755BNX/+nTp5WQkKCwsDDVqFFDvXr1UnZ2tts+MjMzFR8fr2rVqqlOnToaOXKkCgsLyzItAABgoDKFnfJy5ZVX6rnnnlN6erq2bNmim2++WXfddZd27dolSRo+fLiWL1+uJUuW6PPPP9ehQ4fUs2dP1/ZFRUWKj49XQUGBNm7cqPnz52vevHkaO3asp6YEAACqGJtlWdaFbtSlS5c/vVx1IWd3/ltoaKief/553X333apdu7YWLlyou+++W5K0Z88eNWvWTGlpaerYsaM++eQT3XHHHTp06JDCw8Ml/f5Y/KhRo/Trr7/Kz8/vvI7pdDoVHBysvLw82e32MtcOoGrbm31cYz/a+dcDAVy05LtaqEl4kKfLkFTGMztt2rRR69atXUt0dLQKCgq0detWtWzZskyFFBUVadGiRTp58qQcDofS09N15swZxcbGusZERUUpMjJSaWlpkqS0tDS1bNnSFXQkKS4uTk6n03V26Gzy8/PldDrdFgAAYKYy3aA8bdq0s7aPHz9eJ06cuKB97dixQw6HQ6dPn1aNGjX04YcfKjo6Wtu2bZOfn59CQkLcxoeHhysrK0uSlJWV5RZ0SvpL+s5l8uTJmjBhwgXVCQAALk3les/Offfdd8G/F6tp06batm2bvvrqKz3yyCMaOHCgvvvuu/Isq5SkpCTl5eW5loMHD1bo8QAAgOeU+ReBnk1aWpoCAgIuaBs/Pz9dc801kqSYmBht3rxZM2bMUO/evVVQUKDc3Fy3szvZ2dmKiIiQJEVEROjrr79221/J01olY87G399f/v7+F1QnAAC4NJUp7PzxiShJsixLhw8f1pYtWzRmzJiLKqi4uFj5+fmKiYmRr6+vUlNT1atXL0lSRkaGMjMz5XA4JEkOh0OTJk1STk6O6tSpI0lau3at7Ha7oqOjL6oOAABghjKFneDgYLd1Ly8vNW3aVMnJyeratet57ycpKUndunVTZGSkjh8/roULF2r9+vVavXq1goODNWTIEI0YMUKhoaGy2+0aNmyYHA6HOnbsKEnq2rWroqOjNWDAAE2ZMkVZWVkaPXq0EhISOHMDAAAklTHszJ07t1wOnpOTo/vvv1+HDx9WcHCwWrVqpdWrV+vWW2+V9PuN0F5eXurVq5fy8/MVFxen2bNnu7b39vbWihUr9Mgjj8jhcKh69eoaOHCgkpOTy6U+AABw6SvT9+yUSE9P1+7duyVJzZs317XXXltuhVUmvmcHuDzwPTtA5alK37NTpjM7OTk56tOnj9avX++6eTg3N1ddunTRokWLVLt27fKsEQAAoMzK9Oj5sGHDdPz4ce3atUtHjx7V0aNHtXPnTjmdTj322GPlXSMAAECZlenMzqpVq7Ru3To1a9bM1RYdHa2UlJQLukEZAACgopXpzE5xcbF8fX1Ltfv6+qq4uPiiiwIAACgvZQo7N998sx5//HEdOnTI1fbLL79o+PDhuuWWW8qtOAAAgItVprDz0ksvyel0qmHDhrr66qt19dVXq1GjRnI6nZo1a1Z51wgAAFBmZbpnp379+tq6davWrVunPXv2SJKaNWvm9hvKAQAAqoILOrPz6aefKjo6Wk6nUzabTbfeequGDRumYcOGqV27dmrevLm+/PLLiqoVAADggl1Q2Jk+fboeeOCBs37xXnBwsB566CFNnTq13IoDAAC4WBcUdrZv367bbrvtnP1du3ZVenr6RRcFAABQXi4o7GRnZ5/1kfMSPj4++vXXXy+6KAAAgPJyQWHniiuu0M6d5/69Mt9++63q1q170UUBAACUlwsKO7fffrvGjBmj06dPl+o7deqUxo0bpzvuuKPcigMAALhYF/To+ejRo/XBBx+oSZMmSkxMVNOmTSVJe/bsUUpKioqKivTMM89USKEAAABlcUFhJzw8XBs3btQjjzyipKQkWZYlSbLZbIqLi1NKSorCw8MrpFAAAICyuOAvFWzQoIFWrlypY8eOaf/+/bIsS40bN1bNmjUroj4AAICLUqZvUJakmjVrql27duVZCwAAQLkr0+/GAgAAuFQQdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0H08XcLk4faZImUf/4+kygMvGNbVryMvL5ukyAFQBhJ1Kknn0Pxr70U5PlwFcNuYOaq9AP29PlwGgCuAyFgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACM5tGwM3nyZLVr105BQUGqU6eOevTooYyMDLcxp0+fVkJCgsLCwlSjRg316tVL2dnZbmMyMzMVHx+vatWqqU6dOho5cqQKCwsrcyoAAKCK8mjY+fzzz5WQkKBNmzZp7dq1OnPmjLp27aqTJ0+6xgwfPlzLly/XkiVL9Pnnn+vQoUPq2bOnq7+oqEjx8fEqKCjQxo0bNX/+fM2bN09jx471xJQAAEAV49Hfer5q1Sq39Xnz5qlOnTpKT0/XTTfdpLy8PL3++utauHChbr75ZknS3Llz1axZM23atEkdO3bUmjVr9N1332ndunUKDw9XmzZtNHHiRI0aNUrjx4+Xn5+fJ6YGAACqiCp1z05eXp4kKTQ0VJKUnp6uM2fOKDY21jUmKipKkZGRSktLkySlpaWpZcuWCg8Pd42Ji4uT0+nUrl27KrF6AABQFXn0zM4fFRcX64knntD111+vFi1aSJKysrLk5+enkJAQt7Hh4eHKyspyjflj0CnpL+k7m/z8fOXn57vWnU5neU0DAABUMVXmzE5CQoJ27typRYsWVfixJk+erODgYNdSv379Cj8mAADwjCoRdhITE7VixQp99tlnuvLKK13tERERKigoUG5urtv47OxsRUREuMb899NZJeslY/5bUlKS8vLyXMvBgwfLcTYAAKAq8WjYsSxLiYmJ+vDDD/Xpp5+qUaNGbv0xMTHy9fVVamqqqy0jI0OZmZlyOBySJIfDoR07dignJ8c1Zu3atbLb7YqOjj7rcf39/WW3290WAABgJo/es5OQkKCFCxfqo48+UlBQkOsem+DgYAUGBio4OFhDhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6enB4AAKgCPBp25syZI0nq3LmzW/vcuXM1aNAgSdK0adPk5eWlXr16KT8/X3FxcZo9e7ZrrLe3t1asWKFHHnlEDodD1atX18CBA5WcnFxZ0wAAAFWYR8OOZVl/OSYgIEApKSlKSUk555gGDRpo5cqV5VkaAAAwRJW4QRkAAKCiEHYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBoHg07X3zxhbp376569erJZrNp6dKlbv2WZWns2LGqW7euAgMDFRsbq3379rmNOXr0qPr37y+73a6QkBANGTJEJ06cqMRZAACAqsyjYefkyZNq3bq1UlJSzto/ZcoUzZw5Uy+//LK++uorVa9eXXFxcTp9+rRrTP/+/bVr1y6tXbtWK1as0BdffKEHH3ywsqYAAACqOB9PHrxbt27q1q3bWfssy9L06dM1evRo3XXXXZKkN998U+Hh4Vq6dKn69Omj3bt3a9WqVdq8ebPatm0rSZo1a5Zuv/12vfDCC6pXr16lzQUAAFRNVfaenR9//FFZWVmKjY11tQUHB6tDhw5KS0uTJKWlpSkkJMQVdCQpNjZWXl5e+uqrryq9ZgAAUPV49MzOn8nKypIkhYeHu7WHh4e7+rKyslSnTh23fh8fH4WGhrrGnE1+fr7y8/Nd606ns7zKBgAAVUyVPbNTkSZPnqzg4GDXUr9+fU+XBAAAKkiVDTsRERGSpOzsbLf27OxsV19ERIRycnLc+gsLC3X06FHXmLNJSkpSXl6eazl48GA5Vw8AAKqKKht2GjVqpIiICKWmprranE6nvvrqKzkcDkmSw+FQbm6u0tPTXWM+/fRTFRcXq0OHDufct7+/v+x2u9sCAADM5NF7dk6cOKH9+/e71n/88Udt27ZNoaGhioyM1BNPPKFnn31WjRs3VqNGjTRmzBjVq1dPPXr0kCQ1a9ZMt912mx544AG9/PLLOnPmjBITE9WnTx+exAIAAJI8HHa2bNmiLl26uNZHjBghSRo4cKDmzZunf/zjHzp58qQefPBB5ebm6oYbbtCqVasUEBDg2ubtt99WYmKibrnlFnl5ealXr16aOXNmpc8FAABUTR4NO507d5ZlWefst9lsSk5OVnJy8jnHhIaGauHChRVRHgAAMECVvWcHAACgPBB2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaMaEnZSUFDVs2FABAQHq0KGDvv76a0+XBAAAqgAjws7ixYs1YsQIjRs3Tlu3blXr1q0VFxennJwcT5cGAAA8zGZZluXpIi5Whw4d1K5dO7300kuSpOLiYtWvX1/Dhg3T008//ZfbO51OBQcHKy8vT3a7vUJqPH2mSJlH/1Mh+wZQ2jW1a8jLy+bWxucQqDyRodUU4Ovt6TIkST6eLuBiFRQUKD09XUlJSa42Ly8vxcbGKi0tzYOVuQvw9VaT8CBPlwFc1vgcApenSz7s/PbbbyoqKlJ4eLhbe3h4uPbs2XPWbfLz85Wfn+9az8vLk/T7GR4AAHBpCQoKks1mO2f/JR92ymLy5MmaMGFCqfb69et7oBoAAHAx/uo2lEs+7NSqVUve3t7Kzs52a8/OzlZERMRZt0lKStKIESNc68XFxTp69KjCwsL+NBni8uN0OlW/fn0dPHiwwu7nAnBufAZxPoKC/vzy9CUfdvz8/BQTE6PU1FT16NFD0u/hJTU1VYmJiWfdxt/fX/7+/m5tISEhFVwpLmV2u52/aAEP4jOIi3HJhx1JGjFihAYOHKi2bduqffv2mj59uk6ePKnBgwd7ujQAAOBhRoSd3r1769dff9XYsWOVlZWlNm3aaNWqVaVuWgYAAJcfI8KOJCUmJp7zshVQVv7+/ho3blypy54AKgefQZQHI75UEAAA4FyM+HURAAAA50LYAQAARiPsAAAAoxF2cNnp3LmznnjiiXLd5/r162Wz2ZSbm1uu+wVwcRo2bKjp06d7ugx4GGEHAAAYjbADAACMRtjBZamwsFCJiYkKDg5WrVq1NGbMGJV8C8Nbb72ltm3bKigoSBEREerXr59ycnLctl+5cqWaNGmiwMBAdenSRQcOHPDALIBLx/Hjx9W/f39Vr15ddevW1bRp09wuKR87dkz333+/atasqWrVqqlbt27at2+f2z7ef/99NW/eXP7+/mrYsKFefPFFt/6cnBx1795dgYGBatSokd5+++3Kmh6qOMIOLkvz58+Xj4+Pvv76a82YMUNTp07Va6+9Jkk6c+aMJk6cqO3bt2vp0qU6cOCABg0a5Nr24MGD6tmzp7p3765t27Zp6NChevrppz00E+DSMGLECG3YsEHLli3T2rVr9eWXX2rr1q2u/kGDBmnLli1atmyZ0tLSZFmWbr/9dp05c0aSlJ6ernvvvVd9+vTRjh07NH78eI0ZM0bz5s1z28fBgwf12Wef6b333tPs2bNL/aCCy5QFXGY6depkNWvWzCouLna1jRo1ymrWrNlZx2/evNmSZB0/ftyyLMtKSkqyoqOj3caMGjXKkmQdO3aswuoGLlVOp9Py9fW1lixZ4mrLzc21qlWrZj3++OPW3r17LUnWhg0bXP2//fabFRgYaL377ruWZVlWv379rFtvvdVtvyNHjnR9FjMyMixJ1tdff+3q3717tyXJmjZtWgXODpcCzuzgstSxY0fZbDbXusPh0L59+1RUVKT09HR1795dkZGRCgoKUqdOnSRJmZmZkqTdu3erQ4cObvtzOByVVzxwifnhhx905swZtW/f3tUWHByspk2bSvr9M+Xj4+P2uQoLC1PTpk21e/du15jrr7/ebb/XX3+963Nbso+YmBhXf1RUlEJCQipwZrhUEHaAPzh9+rTi4uJkt9v19ttva/Pmzfrwww8lSQUFBR6uDgBQFoQdXJa++uort/VNmzapcePG2rNnj44cOaLnnntON954o6Kiokpd82/WrJm+/vrrUtsDOLurrrpKvr6+2rx5s6stLy9Pe/fulfT7Z6qwsNDtc3nkyBFlZGQoOjraNWbDhg1u+92wYYOaNGkib29vRUVFqbCwUOnp6a7+jIwMvvsKkgg7uExlZmZqxIgRysjI0DvvvKNZs2bp8ccfV2RkpPz8/DRr1iz98MMPWrZsmSZOnOi27cMPP6x9+/Zp5MiRysjI0MKFC91ukgTgLigoSAMHDtTIkSP12WefadeuXRoyZIi8vLxks9nUuHFj3XXXXXrggQf073//W9u3b9d9992nK664QnfddZck6cknn1RqaqomTpyovXv3av78+XrppZf01FNPSZKaNm2q2267TQ899JC++uorpaena+jQoQoMDPTk1FFVePqmIaCyderUyXr00Uethx9+2LLb7VbNmjWt//3f/3XdsLxw4UKrYcOGlr+/v+VwOKxly5ZZkqxvvvnGtY/ly5db11xzjeXv72/deOON1htvvMENysCfcDqdVr9+/axq1apZERER1tSpU6327dtbTz/9tGVZlnX06FFrwIABVnBwsBUYGGjFxcVZe/fuddvHe++9Z0VHR1u+vr5WZGSk9fzzz7v1Hz582IqPj7f8/f2tyMhI680337QaNGjADcqwbJb1/3+5CAAAleTkyZO64oor9OKLL2rIkCGeLgeG8/F0AQAA833zzTfas2eP2rdvr7y8PCUnJ0uS6zIVUJEIOwCASvHCCy8oIyNDfn5+iomJ0ZdffqlatWp5uixcBriMBQAAjMbTWAAAwGiEHQAAYDTCDgAAMBphBwAAGI2wA+CSdeDAAdlsNm3bts3TpQCowgg7AADAaIQdAABgNMIOgCqvuLhYU6ZM0TXXXCN/f39FRkZq0qRJpcYVFRVpyJAhatSokQIDA9W0aVPNmDHDbcz69evVvn17Va9eXSEhIbr++uv1008/SZK2b9+uLl26KCgoSHa7XTExMdqyZUulzBFAxeEblAFUeUlJSfrXv/6ladOm6YYbbtDhw4e1Z8+eUuOKi4t15ZVXasmSJQoLC9PGjRv14IMPqm7durr33ntVWFioHj166IEHHtA777yjgoICff3117LZbJKk/v3769prr9WcOXPk7e2tbdu2ydfXt7KnC6Cc8Q3KAKq048ePq3bt2nrppZc0dOhQt74DBw6oUaNG+uabb9SmTZuzbp+YmKisrCy99957Onr0qMLCwrR+/Xp16tSp1Fi73a5Zs2Zp4MCBFTEVAB7CZSwAVdru3buVn5+vW2655bzGp6SkKCYmRrVr11aNGjX06quvKjMzU5IUGhqqQYMGKS4uTt27d9eMGTN0+PBh17YjRozQ0KFDFRsbq+eee07ff/99hcwJQOUi7ACo0gIDA8977KJFi/TUU09pyJAhWrNmjbZt26bBgweroKDANWbu3LlKS0vTddddp8WLF6tJkybatGmTJGn8+PHatWuX4uPj9emnnyo6Oloffvhhuc8JQOXiMhaAKu306dMKDQ3VzJkz//Iy1rBhw/Tdd98pNTXVNSY2Nla//fbbOb+Lx+FwqF27dpo5c2apvr59++rkyZNatmxZuc4JQOXizA6AKi0gIECjRo3SP/7xD7355pv6/vvvtWnTJr3++uulxjZu3FhbtmzR6tWrtXfvXo0ZM0abN2929f/4449KSkpSWlqafvrpJ61Zs0b79u1Ts2bNdOrUKSUmJmr9+vX66aeftGHDBm3evFnNmjWrzOkCqAA8jQWgyhszZox8fHw0duxYHTp0SHXr1tXDDz9catxDDz2kb775Rr1795bNZlPfvn316KOP6pNPPpEkVatWTXv27NH8+fN15MgR1a1bVwkJCXrooYdUWFioI0eO6P7771d2drZq1aqlnj17asKECZU9XQDljMtYAADAaFzGAgAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBo/x9dWkm/NZ32CAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# plot the target variable \"class\"\n", + "p = sns.histplot(train[\"class\"], ec=\"w\", lw=4)\n", + "_ = p.set_title(\"Bad vs. Good Loan Count\")\n", + "_ = p.spines[\"top\"].set_visible(False)\n", + "_ = p.spines[\"right\"].set_visible(False)" + ] + }, + { + "cell_type": "markdown", + "id": "c6a697a5-5709-4a69-b644-62779b4f8bc5", + "metadata": {}, + "source": [ + "Now, view the first few records of the context data." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "79424785-129d-4007-84a5-041b6d38457d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestampdurationcredit_amountinstallment_commitmentchecking_statusresidence_sinceageexisting_creditsnum_dependentshousing
18473good2023-12-16 03:29:12+00:00612384no checking43612own
764894good2023-11-15 23:19:35+00:001811694no checking32921own
504318good2023-11-23 13:03:53+00:00127014no checking23221own
454340good2023-12-26 17:59:37+00:0024574320<=X<20042421for free
453605good2023-12-18 11:27:02+00:002428284<042211own
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp duration credit_amount \\\n", + "18 473 good 2023-12-16 03:29:12+00:00 6 1238 \n", + "764 894 good 2023-11-15 23:19:35+00:00 18 1169 \n", + "504 318 good 2023-11-23 13:03:53+00:00 12 701 \n", + "454 340 good 2023-12-26 17:59:37+00:00 24 5743 \n", + "453 605 good 2023-12-18 11:27:02+00:00 24 2828 \n", + "\n", + " installment_commitment checking_status residence_since age \\\n", + "18 4 no checking 4 36 \n", + "764 4 no checking 3 29 \n", + "504 4 no checking 2 32 \n", + "454 2 0<=X<200 4 24 \n", + "453 4 <0 4 22 \n", + "\n", + " existing_credits num_dependents housing \n", + "18 1 2 own \n", + "764 2 1 own \n", + "504 2 1 own \n", + "454 2 1 for free \n", + "453 1 1 own " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# View first records in training data\n", + "train.head()" + ] + }, + { + "cell_type": "markdown", + "id": "fd52f5bc-aa0f-48db-b356-c52aa7ce3724", + "metadata": {}, + "source": [ + "### Feature Engineering" + ] + }, + { + "cell_type": "markdown", + "id": "3e5b5c02-ad4d-400e-bdac-bfdf2799f575", + "metadata": {}, + "source": [ + "Once data columns have been prepared so that they can be used to train an AI model, it is common to refer to them as \"features\". The process of preparing features is referred to as \"feature engineering\". \n", + "\n", + "Below, we will train a random forest model. Random forests are relatively robust to non-standardized, non-normalized data, making it easier for us to getting started. As such, the numerical columns are ready for a simple baseline training. \n", + "\n", + "We have pulled two categorical columns, wich we will need to engineer into numerical features." + ] + }, + { + "cell_type": "markdown", + "id": "45a6fb27-140c-4f5a-b464-1f5e5d81d086", + "metadata": {}, + "source": [ + "The `checking_status` column tells us roughly how much money the applicant has in their checking account, while the `housing` column shows the applicant's housing status. We presume that more money in checking correlates inversely with credit risk, while owing vs. renting, vs. living for free correlates directly with credit risk. Hence, converting these to ordinal features makes sense. Of course, in a real study we would want to quantitatively verify these presumptions." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9e374096-b02d-4cbb-8fca-dcc451c90c50", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "checking_status\n", + "no checking 0.39375\n", + "0<=X<200 0.27500\n", + "<0 0.26125\n", + ">=200 0.07000\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `checking_status` column distibution\n", + "train.checking_status.value_counts(normalize=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0144b525-244b-4526-8e4b-d393cb174d06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "housing\n", + "own 0.7225\n", + "rent 0.1675\n", + "for free 0.1100\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `housing` column distribution\n", + "train.housing.value_counts(normalize=True)" + ] + }, + { + "cell_type": "markdown", + "id": "2cb340b4-7d21-4810-8be2-1633da2e4396", + "metadata": {}, + "source": [ + "We define a tranformer that can be used to convert `checking_status` and `housing` to ordinal variables. The transformer will also drop the non-feature columns (`class`, `ID`, and `application_timestamp`) from the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "27796e23-c12e-4e51-8fb4-090b26aff2ef", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature lists\n", + "cat_features = [\"checking_status\", \"housing\"]\n", + "num_features = [\n", + " \"duration\", \"credit_amount\", \"installment_commitment\",\n", + " \"residence_since\", \"age\", \"existing_credits\", \"num_dependents\"\n", + "]\n", + "\n", + "# Ordinal encoder for cat_features\n", + "# (We use a ColumnTransformer to passthrough numerical feature columns)\n", + "col_transform = ColumnTransformer([\n", + " (\"cat_features\", OrdinalEncoder(), cat_features),\n", + " (\"num_features\", \"passthrough\", num_features),\n", + " ],\n", + " remainder=\"drop\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "318429b9-e008-4cc7-8108-779934f9ac2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
checking_statushousingdurationcredit_amountinstallment_commitmentresidence_sinceageexisting_creditsnum_dependents
183.01.06.01238.04.04.036.01.02.0
7643.01.018.01169.04.03.029.02.01.0
5043.01.012.0701.04.02.032.02.01.0
4540.00.024.05743.02.04.024.02.01.0
4531.01.024.02828.04.04.022.01.01.0
\n", + "
" + ], + "text/plain": [ + " checking_status housing duration credit_amount \\\n", + "18 3.0 1.0 6.0 1238.0 \n", + "764 3.0 1.0 18.0 1169.0 \n", + "504 3.0 1.0 12.0 701.0 \n", + "454 0.0 0.0 24.0 5743.0 \n", + "453 1.0 1.0 24.0 2828.0 \n", + "\n", + " installment_commitment residence_since age existing_credits \\\n", + "18 4.0 4.0 36.0 1.0 \n", + "764 4.0 3.0 29.0 2.0 \n", + "504 4.0 2.0 32.0 2.0 \n", + "454 2.0 4.0 24.0 2.0 \n", + "453 4.0 4.0 22.0 1.0 \n", + "\n", + " num_dependents \n", + "18 2.0 \n", + "764 1.0 \n", + "504 1.0 \n", + "454 1.0 \n", + "453 1.0 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the tranform outputs features as expected\n", + "# (Note: transform output is an array, so we convert it\n", + "# back to dataframe for inspection)\n", + "pd.DataFrame(\n", + " index=train.index,\n", + " columns=cat_features + num_features,\n", + " data= col_transform.fit_transform(train)\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "a3785c93-8830-4fa2-bb9d-31b6e8fecb01", + "metadata": {}, + "source": [ + "Finally, let's separate out the labels, and engineer them from categorical (\"good\" | \"bad\") to float (1.0 | 0.0). We do this for both the training and validation data." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "30ebff90-a193-43a2-86fb-cf09e7d03777", + "metadata": {}, + "outputs": [], + "source": [ + "# Make \"class\" target variable numeric\n", + "train_y = (train[\"class\"] == \"good\").astype(float)\n", + "validate_y = (validate[\"class\"] == \"good\").astype(float)" + ] + }, + { + "cell_type": "markdown", + "id": "b052f6b2-2a34-441d-8a5f-2aad4e4db022", + "metadata": {}, + "source": [ + "### Train the Model" + ] + }, + { + "cell_type": "markdown", + "id": "c4f14590-31f4-4680-b1a1-75755a78513e", + "metadata": {}, + "source": [ + "Now that the features are prepared, we can train (fit) our baseline model on the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "0ff48f34-dbb6-4221-aefc-3c9b3f9da3e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the model\n", + "rf_model = RandomForestClassifier(\n", + " n_estimators=400,\n", + " criterion=\"entropy\",\n", + " max_depth=4,\n", + " min_samples_leaf=10,\n", + " class_weight={0:5, 1:1},\n", + " random_state=SEED\n", + ")\n", + "\n", + "# Package transform and model in pipeline\n", + "model = Pipeline([(\"transform\", col_transform), (\"rf_model\", rf_model)])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "1d6ef38a-23b0-4056-a108-960495521164", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Pipeline(steps=[('transform',\n",
+       "                 ColumnTransformer(transformers=[('cat_features',\n",
+       "                                                  OrdinalEncoder(),\n",
+       "                                                  ['checking_status',\n",
+       "                                                   'housing']),\n",
+       "                                                 ('num_features', 'passthrough',\n",
+       "                                                  ['duration', 'credit_amount',\n",
+       "                                                   'installment_commitment',\n",
+       "                                                   'residence_since', 'age',\n",
+       "                                                   'existing_credits',\n",
+       "                                                   'num_dependents'])])),\n",
+       "                ('rf_model',\n",
+       "                 RandomForestClassifier(class_weight={0: 5, 1: 1},\n",
+       "                                        criterion='entropy', max_depth=4,\n",
+       "                                        min_samples_leaf=10, n_estimators=400,\n",
+       "                                        random_state=142))])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "Pipeline(steps=[('transform',\n", + " ColumnTransformer(transformers=[('cat_features',\n", + " OrdinalEncoder(),\n", + " ['checking_status',\n", + " 'housing']),\n", + " ('num_features', 'passthrough',\n", + " ['duration', 'credit_amount',\n", + " 'installment_commitment',\n", + " 'residence_since', 'age',\n", + " 'existing_credits',\n", + " 'num_dependents'])])),\n", + " ('rf_model',\n", + " RandomForestClassifier(class_weight={0: 5, 1: 1},\n", + " criterion='entropy', max_depth=4,\n", + " min_samples_leaf=10, n_estimators=400,\n", + " random_state=142))])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Fit the model\n", + "model.fit(train, train_y)" + ] + }, + { + "cell_type": "markdown", + "id": "73c45c39-9d8e-4f76-aca5-9f0c1568d263", + "metadata": {}, + "source": [ + "### Evaluate the Model" + ] + }, + { + "cell_type": "markdown", + "id": "ef58d432-80ba-428f-b59f-621a9e53b331", + "metadata": {}, + "source": [ + "Let's evaluate our baseline model performance. With credit risk, recall is going to be an important measure to look at. We compare the performance on the training data, with the performance on the validation data through a classification report." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8c5472f6-2ddc-437d-8102-4d5bd2c9f39c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.42 0.92 0.58 232\n", + " 1.0 0.94 0.49 0.64 568\n", + "\n", + " accuracy 0.61 800\n", + " macro avg 0.68 0.70 0.61 800\n", + "weighted avg 0.79 0.61 0.63 800\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate training set performance\n", + "train_preds = model.predict(train)\n", + "print(classification_report(train_y, train_preds))" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "c296bbd3-603e-4615-abbe-2689ebcf5d8c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.46 0.87 0.61 68\n", + " 1.0 0.88 0.48 0.62 132\n", + "\n", + " accuracy 0.61 200\n", + " macro avg 0.67 0.68 0.61 200\n", + "weighted avg 0.74 0.61 0.62 200\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate validation data performance\n", + "print(classification_report(validate_y, model.predict(validate)))" + ] + }, + { + "cell_type": "markdown", + "id": "d57ffbdc-f0b3-4fb6-9575-5acd983082cf", + "metadata": {}, + "source": [ + "The recall on the validation set for bad loans (0 class) is 0.87, meaning that the model correctly identified close to 90% of the bad loans. However, the precision of 0.46 tells us that the model is also classifying many loans that were actually good as bad. Precision and recall are technical metrics. In order to truly assess the models value, we would need feedback from the business side on the impact of misclassifications (for both good and bad loans).\n", + "\n", + "The difference in performance on the training vs. validation data, tells us that the model is slightly overfitting the data. Remember that this is just a quick baseline model. To improve further, we could do things like:\n", + "- gather more data\n", + "- engineer features\n", + "- experiment with hyperparameter settings\n", + "- experiment with other model types\n", + "\n", + "In fact, this is just a start. Creating AI models that meet business needs often requires a lot of guided experimentation." + ] + }, + { + "cell_type": "markdown", + "id": "0378d21a-d6db-42f9-851a-ce71f68c6802", + "metadata": {}, + "source": [ + "### Save the Model" + ] + }, + { + "cell_type": "markdown", + "id": "4450a328-f00c-4579-8e08-b2ebe5046961", + "metadata": {}, + "source": [ + "The last thing we do is save our trained model, so that we can pick it up later in the serving environment." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "da7a7906-d54f-4f2d-9803-6c82c86b28ad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['rf_model.pkl']" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Save the model to a pickle file\n", + "joblib.dump(model, \"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "299588b8-ab67-4155-97a9-770e8e4a7476", + "metadata": {}, + "source": [ + "In the next notebook, [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb), we will load the trained model and request predictions, with input features provided by the Feast online feature server." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb new file mode 100644 index 00000000000..f263dd6cd7b --- /dev/null +++ b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb @@ -0,0 +1,697 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9c870dcb-c66d-454d-a3fa-5f9a723bf8af", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "339ab741-ac90-4763-9971-3b274f6a90b4", + "metadata": {}, + "source": [ + "# Credit Risk Model Serving" + ] + }, + { + "cell_type": "markdown", + "id": "31d29794-4c33-4bc1-9bb4-e238c59f882d", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "d6553fe7-5427-4ecc-b638-615b47acf1a8", + "metadata": {}, + "source": [ + "Model serving is an exciting part of AI/ML. All of our previous work was building to this phase where we can actually serve loan predictions. \n", + "\n", + "So what role does Feast play in model serving? We've already seen that Feast can \"materialize\" data from the training offline store to the serving online store. This comes in handy because many models need contextual features at inference time. \n", + "\n", + "With this example, we can imagine a scenario something like this:\n", + "1. A bank customer submits a loan application on a website. \n", + "2. The website backend requests features, supplying the customer's ID as input.\n", + "3. The backend retrieves feature data for the ID in question.\n", + "4. The backend submits the feature data to the model to obtain a prediction.\n", + "5. The backend uses the prediction to make a decision.\n", + "6. The response is recorded and made available to the user.\n", + "\n", + "With online requests like this, time and resource usage often matter a lot. Feast facilitates quickly retrieving the correct feature data.\n", + "\n", + "In real-life, some of the contextual feature data points could be requested from the user, while others are retrieved from data sources. While outside the scope of this example, Feast does facilitate retrieving request data, and joining it with feature data. (See [Request Source](https://rtd.feast.dev/en/master/#request-source)).\n", + "\n", + "In this notebook, we request feature data from the online store for a small batch of users. We then get outcome predictions from our trained model. This notebook is a continuation of the work done in the previous notebooks; it comes as the step after [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "53818109-c357-435f-8a8b-2a62982fa9a8", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "92b5ab1b-186d-4b76-aac7-9b5110f8673e", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "378189ed-e967-4b2b-b591-aab980a685b3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import joblib\n", + "import json\n", + "import requests\n", + "import warnings\n", + "import pandas as pd\n", + "\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ea90edb2-16f0-4d40-a280-4e6ea79ea5be", + "metadata": {}, + "outputs": [], + "source": [ + "# ingnore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "55f8ed91-7c13-44f7-a294-b6cacd43f8db", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the model\n", + "model = joblib.load(\"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "3093e1b6-66d9-4936-b197-d853631914db", + "metadata": {}, + "source": [ + "### Query Feast Online Server for Feature Data" + ] + }, + { + "cell_type": "markdown", + "id": "2b5bbc4a-e2d3-4b7b-8309-434ff3b3e2cf", + "metadata": {}, + "source": [ + "Here, we show two different ways to retrieve data from the online feature server. The first is using the Python `requests` library, and the second is using the Feast Python SDK.\n", + "\n", + "We can use the Python requests library to request feature data from the online feature server (that we deployed in notebook [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)). The request takes the form of an HTTP POST command sent to the server endpoint (`url`). We request the data we need by supplying the entity and feature information in the data payload. We also need to specify an `application/json` content type in the request header." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c6fd4f1a-917b-4a98-9bf6-101b4a074b64", + "metadata": {}, + "outputs": [], + "source": [ + "# ID examples\n", + "ids = [18, 764, 504, 454, 453, 0, 1, 2, 3, 4, 5, 6, 7, 8]\n", + "\n", + "# Submit get_online_features request to Feast online store server\n", + "response = requests.post(\n", + " url=\"http://localhost:6566/get-online-features\",\n", + " headers = {'Content-Type': 'application/json'},\n", + " data=json.dumps({\n", + " \"entities\": {\"ID\": ids},\n", + " \"features\": [\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + " })\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8e616a52-c18c-44a9-9e63-3aba071d7e79", + "metadata": {}, + "source": [ + "The response is returned as JSON, with feature values for each of the IDs." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cf8948b7-4ed7-4c45-8acf-462331d9e4d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'{\"metadata\":{\"feature_names\":[\"ID\",\"checking_status\",\"duration\",\"installment_commitment\",\"credit_amount\",\"residence_since\",\"num_dependents\",\"age\",\"housing\",\"existing_credits\"]},\"results\":[{\"values\":[18,764,504,454,453,0,1,2,3,4,5,6,7,8],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\"]},{\"values\":[\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"<0\",\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"no checking\",\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show first 1000 characters of response\n", + "response.text[:1000]" + ] + }, + { + "cell_type": "markdown", + "id": "c719f702-578a-4f35-b8ff-e41707cda23e", + "metadata": {}, + "source": [ + "As the response data comes in JSON format, there is a little formatting required to organize the data into a dataframe with one record per row (and features as columns)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b992063d-8d83-4bf7-8153-f690b0410359", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDchecking_statusdurationinstallment_commitmentcredit_amountresidence_sincenum_dependentsagehousingexisting_credits
0180<=X<20024.04.012579.02.01.044.0for free1.0
1764no checking24.04.02463.03.01.027.0own2.0
2504<024.04.01207.04.01.024.0rent1.0
\n", + "
" + ], + "text/plain": [ + " ID checking_status duration installment_commitment credit_amount \\\n", + "0 18 0<=X<200 24.0 4.0 12579.0 \n", + "1 764 no checking 24.0 4.0 2463.0 \n", + "2 504 <0 24.0 4.0 1207.0 \n", + "\n", + " residence_since num_dependents age housing existing_credits \n", + "0 2.0 1.0 44.0 for free 1.0 \n", + "1 3.0 1.0 27.0 own 2.0 \n", + "2 4.0 1.0 24.0 rent 1.0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the response\n", + "resp_data = json.loads(response.text)\n", + "\n", + "# Transform JSON into dataframe\n", + "records = pd.DataFrame(\n", + " columns=resp_data[\"metadata\"][\"feature_names\"], \n", + " data=[[r[\"values\"][i] for r in resp_data[\"results\"]] for i in range(len(ids))]\n", + ")\n", + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "6db9b8ac-146e-40d3-b35a-cf4f4b6bbc8a", + "metadata": {}, + "source": [ + "Now, let's see how we can do the same with the Feast Python SDK. Note that we instantiate our `FeatureStore` object with the configuration that we set up in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb), by pointing to the `./Feature_Store` directory." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "765dc62b-e1e7-45fe-88b4-cc0235519ff8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "WARNING:root:Cannot use sqlite_vec for vector search\n" + ] + } + ], + "source": [ + "# Instantiate FeatureStore object\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "\n", + "# Retrieve features\n", + "records = store.get_online_features(\n", + " entity_rows=[{\"ID\":v} for v in ids],\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\" \n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1d214e55-df0b-460d-936c-8951f7365a93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDcredit_amountinstallment_commitmentchecking_statusdurationnum_dependentshousingageresidence_sinceexisting_credits
01812579.04.00<=X<20024.01.0for free44.02.01.0
17642463.04.0no checking24.01.0own27.03.02.0
25041207.04.0<024.01.0rent24.04.01.0
\n", + "
" + ], + "text/plain": [ + " ID credit_amount installment_commitment checking_status duration \\\n", + "0 18 12579.0 4.0 0<=X<200 24.0 \n", + "1 764 2463.0 4.0 no checking 24.0 \n", + "2 504 1207.0 4.0 <0 24.0 \n", + "\n", + " num_dependents housing age residence_since existing_credits \n", + "0 1.0 for free 44.0 2.0 1.0 \n", + "1 1.0 own 27.0 3.0 2.0 \n", + "2 1.0 rent 24.0 4.0 1.0 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "fd828758-6c57-4f9e-bbda-3983b6579da2", + "metadata": {}, + "source": [ + "### Get Predictions from the Model" + ] + }, + { + "cell_type": "markdown", + "id": "f446d7ec-0dae-409a-82a2-c0d7016c2001", + "metadata": {}, + "source": [ + "Now we can request predictions from our trained model. \n", + "\n", + "For convenience, we output the predictions along with the implied loan designations. Remember that these are predictions on loan outcomes, given context data from the loan application process. Since we have access to the actual `class` outcomes, we display those as well to see how the model did.|" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "70203f7b-f1e5-46ba-8623-f10bf3a5abf8", + "metadata": {}, + "outputs": [], + "source": [ + "# Get predictions from the model\n", + "preds = model.predict(records)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "27001dde-8bdb-4de1-8c33-a76f030748e0", + "metadata": {}, + "outputs": [], + "source": [ + "# Load labels\n", + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ddc958e8-8ff8-49b1-ac10-fc965f3bf21c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDPredictionLoan_DesignationTrue_Value
18180.0badbad
7647641.0goodgood
5045040.0badbad
4544540.0badbad
4534531.0goodgood
001.0goodgood
110.0badbad
221.0goodgood
330.0badgood
440.0badbad
551.0goodgood
661.0goodgood
770.0badgood
881.0goodgood
\n", + "
" + ], + "text/plain": [ + " ID Prediction Loan_Designation True_Value\n", + "18 18 0.0 bad bad\n", + "764 764 1.0 good good\n", + "504 504 0.0 bad bad\n", + "454 454 0.0 bad bad\n", + "453 453 1.0 good good\n", + "0 0 1.0 good good\n", + "1 1 0.0 bad bad\n", + "2 2 1.0 good good\n", + "3 3 0.0 bad good\n", + "4 4 0.0 bad bad\n", + "5 5 1.0 good good\n", + "6 6 1.0 good good\n", + "7 7 0.0 bad good\n", + "8 8 1.0 good good" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show preds\n", + "pd.DataFrame({\n", + " \"ID\": ids,\n", + " \"Prediction\": preds,\n", + " \"Loan_Designation\": [\"bad\" if i==0.0 else \"good\" for i in preds],\n", + " \"True_Value\": labels.loc[ids, \"class\"]\n", + "})" + ] + }, + { + "cell_type": "markdown", + "id": "87cd592a-61fc-4553-b84a-941d1785910d", + "metadata": {}, + "source": [ + "It's important to remember that the model's predictions are like educated guesses based on learned patterns. The model will get some predictions right, and other wrong. With the example records above, it looks like the model did pretty good! An AI/ML team's task is generally to make the model's predictions as useful as possible in helping the organization make decisions (for example, on loan approvals).\n", + "\n", + "In this case, we have a baseline model. While not ready for production, this model has set a low bar by which other models can be measured. Teams can also use a model like this to help with early testing, and with proving out things like pipelines and infrastructure before more sophisticated models are available.\n", + "\n", + "We have used Feast to query the feature data in support of model serving. The next notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb), cleans up resources created in this and previous notebooks." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb new file mode 100644 index 00000000000..846748dc425 --- /dev/null +++ b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cf46ec61-7914-4677-b12b-a9e478e88d3f", + "metadata": {}, + "source": [ + "# Credit Risk Cleanup" + ] + }, + { + "cell_type": "markdown", + "id": "6ae8aaec-e01d-48d3-b768-98661ad1ec85", + "metadata": {}, + "source": [ + "Run this notebook if you are done experimenting with this demo, or if you wish to start again with a clean slate.\n", + "\n", + "**RUNNING THE FOLLOWING CODE WILL REMOVE FILES AND PROCESSES CREATED BY THE PREVIOUS EXAMPLE NOTEBOOKS.**\n", + "\n", + "The notebook progresses in reverse order of how the files and processes were added. (The reverse order makes it possible to partially revert changes by running cells up to a certain point.)" + ] + }, + { + "cell_type": "markdown", + "id": "6feaa771-4226-459f-b6dd-214024cb5c7c", + "metadata": {}, + "source": [ + "#### Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "20a39e94-920d-4108-aa6b-1e29d2224f71", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import time\n", + "import psutil" + ] + }, + { + "cell_type": "markdown", + "id": "3f124260-a8b2-475d-9103-8d336c543fce", + "metadata": {}, + "source": [ + "#### Remove Trained Model File" + ] + }, + { + "cell_type": "markdown", + "id": "f7a05a2b-9a26-4722-a526-84da99fc0b29", + "metadata": {}, + "source": [ + "This removes the model that was created and saved in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a6b21063-ea43-4329-be0c-c1644c705db2", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the model file that was saved in model training.\n", + "model_path = \"./rf_model.pkl\"\n", + "os.remove(model_path)" + ] + }, + { + "cell_type": "markdown", + "id": "ed97c24a-8f25-4e77-9037-f9cf4ad68dfa", + "metadata": {}, + "source": [ + "#### Shutdown Servers" + ] + }, + { + "cell_type": "markdown", + "id": "2f825d10-c13d-4701-b102-e15ad1c0bd3b", + "metadata": {}, + "source": [ + "Shut down the servers that were launched in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb); also remove the `server_proc.txt` that held the process PIDs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "66db4d46-a895-4041-ad87-ab0a77f13211", + "metadata": {}, + "outputs": [], + "source": [ + "# Load server process objects\n", + "server_pids = open(\"server_proc.txt\").readlines()\n", + "offline_server_proc = psutil.Process(int(server_pids[0].strip()))\n", + "online_server_proc = psutil.Process(int(server_pids[1].strip()))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "001fd472-2e28-499e-9eac-0a16ad8187a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='running', started='14:19:05')\n", + "Online server is running: True\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='running', started='14:19:03')\n", + "Offline server is running: True\n" + ] + } + ], + "source": [ + "# Verify if servers are running\n", + "def verify_servers():\n", + " # online server\n", + " print(f\"Online server : {online_server_proc}\")\n", + " print(f\"Online server is running: {online_server_proc.is_running()}\", end='\\n\\n')\n", + " # offline server\n", + " print(f\"Offline server PID: {offline_server_proc}\")\n", + " print(f\"Offline server is running: {offline_server_proc.is_running()}\")\n", + " \n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "68376350-790a-4e7e-9325-c7de4d22e54b", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate offline server\n", + "offline_server_proc.terminate()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "446b6bf9-aef2-4873-b477-8bf595a8eabf", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate online server (master and worker)\n", + "for child in online_server_proc.children(recursive=True):\n", + " child.terminate()\n", + "online_server_proc.terminate()\n", + "time.sleep(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "774827f6-4dcd-495b-b5c5-186b97148619", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='terminated', started='14:19:05')\n", + "Online server is running: False\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='terminated', started='14:19:03')\n", + "Offline server is running: False\n" + ] + } + ], + "source": [ + "# Verify termination\n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f8a155e4-23b3-4fb3-b868-02ba2e0a4a31", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove server_proc.txt (file for keeping track of pids)\n", + "os.remove(\"server_proc.txt\")" + ] + }, + { + "cell_type": "markdown", + "id": "ed7d6f25-d255-4986-9cf2-9876f6c558cc", + "metadata": {}, + "source": [ + "#### Remove Feast Applied Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "d73efe15-a1d9-459b-8142-835dc2bf1c9f", + "metadata": {}, + "source": [ + "Remove the registry and online store (SQLite) files created on`feast apply` created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0f13a4ac-d2ad-462b-b65e-4266b7cb4922", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/data/online_store.db\")\n", + "os.remove(\"Feature_Store/data/registry.db\")" + ] + }, + { + "cell_type": "markdown", + "id": "eb0494cd-0143-4f5f-b7d6-9675e1403d9f", + "metadata": {}, + "source": [ + "#### Remove Feast Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "86c33ac7-9e1f-4798-9f14-77773a1c13bd", + "metadata": {}, + "source": [ + "Remove the configution and feature definition files created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a747043f-05fe-4b44-979d-9b30565074ee", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/feature_store.yaml\")\n", + "os.remove(\"Feature_Store/feature_definitions.py\")" + ] + }, + { + "cell_type": "markdown", + "id": "81975a0f-7fd6-4ed3-91cf-812946df4713", + "metadata": {}, + "source": [ + "#### Remove Data Files" + ] + }, + { + "cell_type": "markdown", + "id": "8182dc1e-d5c1-4739-b7c7-0620e93c5b64", + "metadata": {}, + "source": [ + "Remove the data files created in [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4ddb4fb2-fea1-4b70-8978-732af9a1cd3f", + "metadata": {}, + "outputs": [], + "source": [ + "for f in [\"data_a.parquet\", \"data_b.parquet\", \"labels.parquet\"]:\n", + " os.remove(f\"Feature_Store/data/{f}\")\n", + "os.rmdir(\"Feature_Store/data\")\n", + "os.rmdir(\"Feature_Store\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/README.md b/examples/credit-risk-end-to-end/README.md new file mode 100644 index 00000000000..5f59c750784 --- /dev/null +++ b/examples/credit-risk-end-to-end/README.md @@ -0,0 +1,39 @@ + +![Feast_Logo](https://raw.githubusercontent.com/feast-dev/feast/master/docs/assets/feast_logo.png) + +# Feast Credit Risk Classification End-to-End Example + +This example starts with an [OpenML](https://openml.org) credit risk dataset, and walks through the steps of preparing the data, setting up feature store resources, and serving features; this is all done inside the paradigm of an ML workflow, with the goal of helping users understand how Feast fits in the progression from data preparation, to model training and model serving. + +The example is organized in five notebooks: +1. [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) +2. [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) +3. [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb) +4. [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) +5. [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb) + +Run the notebooks in order to progress through the example. See below for prerequisite setup steps. + +### Preparing your Environment +To run the example, install the Python dependencies. You may wish to do so inside a virtual environment. Open a command terminal, and run the following: + +``` +# create venv-example virtual environment +python -m venv venv-example +# activate environment +source venv-example/bin/activate +``` + +Install the Python dependencies: +``` +pip install -r requirements.txt +``` + +Note that this example was tested with Python 3.11, but it should also work with other similar versions. + +### Running the Notebooks +Once you have installed the Python dependencies, you can run the example notebooks. To run the notebooks locally, execute the following command in a terminal window: + +```jupyter notebook``` + +You should see a browser window open a page where you can navigate to the example notebook (.ipynb) files and open them. diff --git a/examples/credit-risk-end-to-end/requirements.txt b/examples/credit-risk-end-to-end/requirements.txt new file mode 100644 index 00000000000..8b9b1313e78 --- /dev/null +++ b/examples/credit-risk-end-to-end/requirements.txt @@ -0,0 +1,6 @@ +feast +jupyter==1.1.1 +scikit-learn==1.5.2 +pandas==2.2.3 +matplotlib==3.9.2 +seaborn==0.13.2 \ No newline at end of file From f95e54bdbee80be6b0e290a02e56f92daac2cf64 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 20 Dec 2024 14:28:58 -0500 Subject: [PATCH 49/90] feat: Adding Feature Server to components docs (#4868) Signed-off-by: Francisco Javier Arceo --- docs/SUMMARY.md | 1 + docs/getting-started/components/README.md | 4 ++ .../components/feature-server.md | 40 +++++++++++++++++++ docs/getting-started/components/overview.md | 2 + 4 files changed, 47 insertions(+) create mode 100644 docs/getting-started/components/feature-server.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 7aad7a94428..e24e15fb5cb 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -32,6 +32,7 @@ * [Registry](getting-started/components/registry.md) * [Offline store](getting-started/components/offline-store.md) * [Online store](getting-started/components/online-store.md) + * [Feature server](getting-started/components/feature-server.md) * [Batch Materialization Engine](getting-started/components/batch-materialization-engine.md) * [Provider](getting-started/components/provider.md) * [Authorization Manager](getting-started/components/authz_manager.md) diff --git a/docs/getting-started/components/README.md b/docs/getting-started/components/README.md index e1c000abced..4c6f3a54dfc 100644 --- a/docs/getting-started/components/README.md +++ b/docs/getting-started/components/README.md @@ -12,6 +12,10 @@ [online-store.md](online-store.md) {% endcontent-ref %} +{% content-ref url="feature-server.md" %} +[feature-server.md](feature-server.md) +{% endcontent-ref %} + {% content-ref url="batch-materialization-engine.md" %} [batch-materialization-engine.md](batch-materialization-engine.md) {% endcontent-ref %} diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md new file mode 100644 index 00000000000..90e6d25e5a2 --- /dev/null +++ b/docs/getting-started/components/feature-server.md @@ -0,0 +1,40 @@ +# Feature Server + +The Feature Server is a core architectural component in Feast, designed to provide low-latency feature retrieval and updates for machine learning applications. + +It is a REST API server built using [FastAPI](https://fastapi.tiangolo.com/) and exposes a limited set of endpoints to serve features, push data, and support materialization operations. The server is scalable, flexible, and designed to work seamlessly with various deployment environments, including local setups and cloud-based systems. + +## Motivation + +In machine learning workflows, real-time access to feature values is critical for enabling low-latency predictions. The Feature Server simplifies this requirement by: + +1. **Serving Features:** Allowing clients to retrieve feature values for specific entities in real-time, reducing the complexity of direct interactions with the online store. +2. **Data Integration:** Providing endpoints to push feature data directly into the online or offline store, ensuring data freshness and consistency. +3. **Scalability:** Supporting horizontal scaling to handle high request volumes efficiently. +4. **Standardized API:** Exposing HTTP/JSON endpoints that integrate seamlessly with various programming languages and ML pipelines. +5. **Secure Communication:** Supporting TLS (SSL) for secure data transmission in production environments. + +## Architecture + +The Feature Server operates as a stateless service backed by two key components: + +- **[Online Store](./online-store.md):** The primary data store used for low-latency feature retrieval. +- **[Registry](./registry.md):** The metadata store that defines feature sets, feature views, and their relationships to entities. + +## Key Features + +1. **RESTful API:** Provides standardized endpoints for feature retrieval and data pushing. +2. **CLI Integration:** Easily managed through the Feast CLI with commands like `feast serve`. +3. **Flexible Deployment:** Can be deployed locally, via Docker, or on Kubernetes using Helm charts. +4. **Scalability:** Designed for distributed deployments to handle large-scale workloads. +5. **TLS Support:** Ensures secure communication in production setups. + +## Endpoints Overview + +| Endpoint | Description | +| -------------------------- | ----------------------------------------------------------------------- | +| `/get-online-features` | Retrieves feature values for specified entities and feature references. | +| `/push` | Pushes feature data to the online and/or offline store. | +| `/materialize` | Materializes features within a specific time range to the online store. | +| `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | + diff --git a/docs/getting-started/components/overview.md b/docs/getting-started/components/overview.md index ac0b99de8ab..05c7503d842 100644 --- a/docs/getting-started/components/overview.md +++ b/docs/getting-started/components/overview.md @@ -13,6 +13,7 @@ * **Deploy Model:** The trained model binary (and list of features) are deployed into a model serving system. This step is not executed by Feast. * **Prediction:** A backend system makes a request for a prediction from the model serving service. * **Get Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. +* **Feature Retrieval:** The online serving service retrieves the latest feature values from the online store and returns them to the model serving service. ## Components @@ -24,6 +25,7 @@ A complete Feast deployment contains the following components: * Materialize (load) feature values into the online store. * Build and retrieve training datasets from the offline store. * Retrieve online features. +* **Feature Server:** The Feature Server is a REST API server that serves feature values for a given entity key and feature reference. The Feature Server is designed to be horizontally scalable and can be deployed in a distributed manner. * **Stream Processor:** The Stream Processor can be used to ingest feature data from streams and write it into the online or offline stores. Currently, there's an experimental Spark processor that's able to consume data from Kafka. * **Batch Materialization Engine:** The [Batch Materialization Engine](batch-materialization-engine.md) component launches a process which loads data into the online store from the offline store. By default, Feast uses a local in-process engine implementation to materialize data. However, additional infrastructure can be used for a more scalable materialization process. * **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is either populated through materialization jobs or through [stream ingestion](../../reference/data-sources/push.md). From 88854dd56fd0becf4a5d5293735a1c9ba394d53d Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Fri, 20 Dec 2024 15:15:25 -0600 Subject: [PATCH 50/90] fix: Refactor Operator to deploy all feast services to the same Deployment/Pod (#4863) * single deployment refactor Signed-off-by: Tommy Hughes * fix e2e tests Signed-off-by: Tommy Hughes --------- Signed-off-by: Tommy Hughes --- .../api/v1alpha1/featurestore_types.go | 28 +- .../api/v1alpha1/zz_generated.deepcopy.go | 25 +- .../crd/bases/feast.dev_featurestores.yaml | 52 +-- ...v1alpha1_featurestore_pvc_persistence.yaml | 5 +- infra/feast-operator/dist/install.yaml | 52 +-- .../internal/controller/authz/authz.go | 27 +- .../featurestore_controller_db_store_test.go | 144 +++---- .../featurestore_controller_ephemeral_test.go | 171 +++----- ...restore_controller_kubernetes_auth_test.go | 195 +++------ .../featurestore_controller_loglevel_test.go | 61 +-- ...eaturestore_controller_objectstore_test.go | 158 ++----- .../featurestore_controller_oidc_auth_test.go | 162 +++---- .../featurestore_controller_pvc_test.go | 262 +++++------ .../featurestore_controller_test.go | 408 ++++++------------ .../featurestore_controller_tls_test.go | 172 +++----- .../internal/controller/services/client.go | 6 +- .../controller/services/repo_config.go | 181 +++++--- .../controller/services/repo_config_test.go | 243 +++-------- .../internal/controller/services/services.go | 365 +++++++++------- .../controller/services/services_types.go | 20 +- .../internal/controller/services/tls.go | 39 +- .../internal/controller/services/tls_test.go | 51 +-- .../internal/controller/services/util.go | 144 +++++-- infra/feast-operator/test/e2e/e2e_test.go | 26 +- infra/feast-operator/test/e2e/test_util.go | 6 +- 25 files changed, 1208 insertions(+), 1795 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 2eb9ec8554d..f73c7fc6a40 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -65,7 +65,7 @@ type FeatureStoreSpec struct { AuthzConfig *AuthzConfig `json:"authz,omitempty"` } -// FeatureStoreServices defines the desired feast service deployments. ephemeral registry is deployed by default. +// FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. type FeatureStoreServices struct { OfflineStore *OfflineStore `json:"offlineStore,omitempty"` OnlineStore *OnlineStore `json:"onlineStore,omitempty"` @@ -74,9 +74,9 @@ type FeatureStoreServices struct { // OfflineStore configures the deployed offline store service type OfflineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OfflineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the offline store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -127,9 +127,9 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ // OnlineStore configures the deployed online store service type OnlineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OnlineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OnlineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the online store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -146,7 +146,7 @@ type OnlineStorePersistence struct { // OnlineStoreFilePersistence configures the file-based persistence for the offline store service // +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths." // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes." -// +kubebuilder:validation:XValidation:rule="has(self.path) && !self.path.startsWith('s3://') && !self.path.startsWith('gs://')",message="Online store does not support S3 or GS buckets." +// +kubebuilder:validation:XValidation:rule="has(self.path) ? !(self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="Online store does not support S3 or GS buckets." type OnlineStoreFilePersistence struct { Path string `json:"path,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` @@ -235,11 +235,11 @@ type PvcConfig struct { Create *PvcCreate `json:"create,omitempty"` // MountPath within the container at which the volume should be mounted. // Must start by "/" and cannot contain ':'. - MountPath string `json:"mountPath,omitempty"` + MountPath string `json:"mountPath"` } // PvcCreate defines the immutable settings to create a new PVC mounted at the given path. -// The PVC name is the same as the associated deployment name. +// The PVC name is the same as the associated deployment & feast service name. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PvcCreate is immutable" type PvcCreate struct { // AccessModes k8s persistent volume access modes. Defaults to ["ReadWriteOnce"]. @@ -292,14 +292,6 @@ type DefaultConfigs struct { Image *string `json:"image,omitempty"` } -// StoreServiceConfigs k8s deployment settings -type StoreServiceConfigs struct { - // Replicas determines the number of pods for the feast service. - // When Replicas > 1, persistence is recommended. - Replicas *int32 `json:"replicas,omitempty"` - ServiceConfigs `json:",inline"` -} - // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 3241dff775b..f1e05030880 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -273,7 +273,7 @@ func (in *LocalRegistryConfig) DeepCopy() *LocalRegistryConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OfflineStorePersistence) @@ -376,7 +376,7 @@ func (in *OidcAuthz) DeepCopy() *OidcAuthz { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OnlineStorePersistence) @@ -721,27 +721,6 @@ func (in *ServiceHostnames) DeepCopy() *ServiceHostnames { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoreServiceConfigs) DeepCopyInto(out *StoreServiceConfigs) { - *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoreServiceConfigs. -func (in *StoreServiceConfigs) DeepCopy() *StoreServiceConfigs { - if in == nil { - return nil - } - out := new(StoreServiceConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index fd8861cef14..2cab2d8c5d8 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -101,8 +101,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -318,6 +318,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -375,12 +377,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -692,6 +688,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -711,8 +709,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -760,12 +758,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1082,6 +1074,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1369,8 +1363,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1588,6 +1582,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1647,12 +1643,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1967,6 +1957,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1987,8 +1979,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2038,12 +2030,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2368,6 +2354,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml index b7c7412c0f0..15aa46c456c 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml @@ -5,6 +5,7 @@ metadata: spec: feastProject: my_project services: + # demonstrates using a pre-existing PVC onlineStore: persistence: file: @@ -13,6 +14,7 @@ spec: ref: name: online-pvc mountPath: /data/online + # demonstrates specifying a storageClassName and storage size offlineStore: persistence: file: @@ -24,6 +26,7 @@ spec: requests: storage: 5Gi mountPath: /data/offline + # demonstrates letting the Operator create a PVC w/ defaults set registry: local: persistence: @@ -39,7 +42,7 @@ metadata: name: online-pvc spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 435be789e51..cd63b3df8d0 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -109,8 +109,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -326,6 +326,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -383,12 +385,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -700,6 +696,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -719,8 +717,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -768,12 +766,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1090,6 +1082,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1377,8 +1371,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1596,6 +1590,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1655,12 +1651,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1975,6 +1965,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1995,8 +1987,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2046,12 +2038,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2376,6 +2362,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/internal/controller/authz/authz.go b/infra/feast-operator/internal/controller/authz/authz.go index efcae23a4b0..8596d993899 100644 --- a/infra/feast-operator/internal/controller/authz/authz.go +++ b/infra/feast-operator/internal/controller/authz/authz.go @@ -134,28 +134,11 @@ func (authz *FeastAuthorization) initFeastRoleBinding() *rbacv1.RoleBinding { func (authz *FeastAuthorization) setFeastRoleBinding(roleBinding *rbacv1.RoleBinding) error { roleBinding.Labels = authz.getLabels() - roleBinding.Subjects = []rbacv1.Subject{} - if authz.Handler.FeatureStore.Status.Applied.Services.OfflineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OfflineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if authz.Handler.FeatureStore.Status.Applied.Services.OnlineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OnlineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if services.IsLocalRegistry(authz.Handler.FeatureStore) { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.RegistryFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } + roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: services.GetFeastName(authz.Handler.FeatureStore), + Namespace: authz.Handler.FeatureStore.Namespace, + }) roleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "Role", diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 0ee269bda17..0bde0dfd7b9 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -127,7 +127,6 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Context("When deploying a resource with all db storage services", func() { const resourceName = "cr-name" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -206,7 +205,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), @@ -430,16 +429,17 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -529,7 +529,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -550,20 +550,20 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -579,29 +579,29 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineDBPersistenceSnowflakeConfigType, + DBParameters: unmarshallYamlString(snowflakeYamlString), + }, Registry: services.RegistryConfig{ Path: copyMap["path"].(string), RegistryType: services.RegistryDBPersistenceSQLConfigType, DBParameters: dbParametersMap, }, + OnlineStore: services.OnlineStoreConfig{ + Type: onlineType, + DBParameters: unmarshallYamlString(cassandraYamlString), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -610,38 +610,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineDBPersistenceSnowflakeConfigType, - DBParameters: unmarshallYamlString(snowflakeYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -650,25 +628,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Type: onlineType, - DBParameters: unmarshallYamlString(cassandraYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(repoConfigOnline).To(Equal(testConfig)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) // check client config cm := &corev1.ConfigMap{} @@ -682,6 +644,15 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -716,17 +687,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { feast.Handler.FeatureStore = resource // check online config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -736,9 +706,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType - onlineConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType + testConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index a762faa5a21..70ac81a056d 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { const resourceName = "services-ephemeral" const offlineType = "duckdb" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -66,7 +65,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ @@ -199,16 +198,17 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -244,7 +244,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -265,20 +265,21 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -291,28 +292,27 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, Registry: services.RegistryConfig{ RegistryType: services.RegistryFileConfigType, Path: registryPath, }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStorePath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -321,37 +321,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -360,25 +338,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStorePath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -396,12 +356,19 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, + OfflineStore: services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + }, OnlineStore: services.OnlineStoreConfig{ Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: regRemote, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfigClient).To(Equal(clientConfig)) @@ -424,17 +391,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) feast.Handler.FeatureStore = resource - // check registry config - deploy = &appsv1.Deployment{} + // check registry err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -443,21 +409,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStorePath testConfig.Registry.Path = newRegistryPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -466,20 +427,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -489,8 +444,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStorePath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStorePath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 57dd3a290df..57a73eb0eb2 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Context("When deploying a resource with all ephemeral services and Kubernetes authorization", func() { const resourceName = "kubernetes-authorization" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -63,7 +62,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} @@ -125,7 +124,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -134,7 +133,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -188,47 +187,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // check configured Roles for _, roleName := range roles { @@ -277,23 +248,21 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Kind: "Role", Name: feastRole.Name, } - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) - expectedSubject := rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: sa.Name, - Namespace: sa.Namespace, - } - Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) - Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) + expectedSubject := rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: sa.Name, + Namespace: sa.Namespace, } + Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) + Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) By("Updating the user roled and reconciling") resourceNew := resource.DeepCopy() @@ -394,7 +363,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -415,19 +384,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { }, } - // check registry deployment + // check registry deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -436,34 +405,22 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, - }, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + } + testConfig.Registry.RegistryType = services.RegistryFileConfigType + testConfig.AuthzConfig = services.AuthzConfig{ + Type: services.KubernetesAuthType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -472,37 +429,14 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -511,26 +445,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -544,6 +459,15 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -553,10 +477,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, + Registry: regRemote, AuthzConfig: services.AuthzConfig{ Type: services.KubernetesAuthType, }, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go index 70f33486fce..5139e14dd38 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -154,43 +154,26 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("ERROR")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("INFO")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("DEBUG")) }) @@ -229,32 +212,22 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index f4a21a28f17..aff36f338e7 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -46,7 +46,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-object-store" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -68,7 +67,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil @@ -176,39 +175,23 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // update S3 additional args and reconcile resourceNew := resource.DeepCopy() @@ -234,16 +217,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.S3AdditionalKwargs).To(Equal(&newS3AdditionalKwargs)) // check registry deployment - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(1)) }) It("should properly encode a feature_store.yaml config", func() { @@ -290,21 +271,28 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -313,38 +301,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: registryPath, - S3AdditionalKwargs: &s3AdditionalKwargs, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: registryPath, + S3AdditionalKwargs: &s3AdditionalKwargs, } - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -358,17 +321,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // remove S3 additional keywords and reconcile resourceNew := resource.DeepCopy() @@ -386,16 +344,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource // check registry config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -405,27 +361,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) testConfig.Registry.S3AdditionalKwargs = nil - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index eb320c5bb39..08c92a88a97 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -49,7 +49,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { const resourceName = "oidc-authorization" const oidcSecretName = "oidc-secret" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -74,7 +73,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, @@ -147,7 +146,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -156,7 +155,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -206,47 +205,22 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) // check Feast Role feastRole := &rbacv1.Role{} @@ -268,16 +242,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(err).To(HaveOccurred()) Expect(errors.IsNotFound(err)).To(BeTrue()) - // check ServiceAccounts - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) - } + // check ServiceAccount + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) By("Clearing the OIDC authorization and reconciling") resourceNew := resource.DeepCopy() @@ -328,7 +300,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -349,19 +321,19 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -374,28 +346,27 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + }, Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, + RegistryType: services.RegistryFileConfigType, + Path: services.EphemeralPath + "/" + services.DefaultRegistryPath, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: services.EphemeralPath + "/" + services.DefaultOnlineStorePath, + Type: services.OnlineSqliteConfigType, }, AuthzConfig: expectedServerOidcAuthorizConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -404,35 +375,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -441,23 +391,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) // check client config @@ -472,6 +405,11 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index e64e5cd6245..887d9070efb 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -50,7 +50,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-pvc" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -79,7 +78,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ @@ -260,96 +259,87 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(offlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + name := feast.GetFeastServiceName(services.RegistryFeastType) + regVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(regVol.Name).To(Equal(name)) + Expect(regVol.PersistentVolumeClaim.ClaimName).To(Equal(name)) + + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.VolumeMounts).To(HaveLen(3)) + offlineVolMount := services.GetOfflineVolumeMount(feast.Handler.FeatureStore, offlineContainer.VolumeMounts) + Expect(offlineVolMount.MountPath).To(Equal(offlineStoreMountPath)) + offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) + Expect(offlineVolMount.Name).To(Equal(offlinePvcName)) // check offline pvc pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: offlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) Expect(pvc.Spec.StorageClassName).To(Equal(&storageClassName)) Expect(pvc.Spec.AccessModes).To(Equal(accessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOfflineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(onlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check online + onlinePvcName := feast.GetFeastServiceName(services.OnlineFeastType) + onlineVol := services.GetOnlineVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(onlineVol.Name).To(Equal(onlinePvcName)) + Expect(onlineVol.PersistentVolumeClaim.ClaimName).To(Equal(onlinePvcName)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(3)) + onlineVolMount := services.GetOnlineVolumeMount(feast.Handler.FeatureStore, onlineContainer.VolumeMounts) + Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) + Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) // check online pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(onlinePvcName)) Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOnlineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(registryMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check registry + registryPvcName := feast.GetFeastServiceName(services.RegistryFeastType) + registryVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(registryVol.Name).To(Equal(registryPvcName)) + Expect(registryVol.PersistentVolumeClaim.ClaimName).To(Equal(registryPvcName)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(3)) + registryVolMount := services.GetRegistryVolumeMount(feast.Handler.FeatureStore, registryContainer.VolumeMounts) + Expect(registryVolMount.MountPath).To(Equal(registryMountPath)) + Expect(registryVolMount.Name).To(Equal(registryPvcName)) // check registry pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: registryPvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(registryPvcName)) Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultRegistryStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -375,19 +365,18 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check online deployment deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) // check online pvc is deleted log.FromContext(feast.Handler.Context).Info("Checking deletion of", "PersistentVolumeClaim", deploy.Name) pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) @@ -421,7 +410,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -442,21 +431,22 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -473,25 +463,24 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { RegistryType: services.RegistryFileConfigType, Path: registryMountedPath, }, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStoreMountedPath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -500,37 +489,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -539,25 +507,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStoreMountedPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -571,6 +521,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -610,14 +569,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check registry config deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -626,21 +585,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath testConfig.Registry.Path = newRegistryMountedPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -649,20 +603,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -672,8 +620,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStoreMountedPath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index debd63300b2..71b5d400f87 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -172,15 +172,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) svc := &corev1.Service{} @@ -220,11 +221,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -232,7 +233,7 @@ var _ = Describe("FeatureStore Controller", func() { env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -241,17 +242,8 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfig).To(Equal(testConfig)) + testConfig := feast.GetDefaultRepoConfig() + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -265,17 +257,12 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-test-resource-registry.default.svc.cluster.local:80", - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-test-resource-registry.default.svc.cluster.local:80", } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // change feast project and reconcile resourceNew := resource.DeepCopy() @@ -291,8 +278,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -302,7 +289,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -310,7 +297,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should error on reconcile", func() { @@ -339,11 +326,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -378,17 +365,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.RegistryFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) Expect(cond).ToNot(BeNil()) @@ -404,7 +390,6 @@ var _ = Describe("FeatureStore Controller", func() { Context("When reconciling a resource with all services enabled", func() { const resourceName = "services" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -420,7 +405,7 @@ var _ = Describe("FeatureStore Controller", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -474,7 +459,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -483,7 +468,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -534,16 +519,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -579,12 +564,12 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) saList := corev1.ServiceAccountList{} err = k8sClient.List(ctx, &saList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(saList.Items).To(HaveLen(3)) + Expect(saList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -607,19 +592,20 @@ var _ = Describe("FeatureStore Controller", func() { // check registry config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -628,33 +614,20 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -663,38 +636,16 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-services-registry.default.svc.cluster.local:80", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -703,25 +654,7 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: "feast-services-offline.default.svc.cluster.local", - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -735,6 +668,15 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: "feast-services-offline.default.svc.cluster.local", + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-services-registry.default.svc.cluster.local:80", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -763,8 +705,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -774,7 +716,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -782,7 +724,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should properly set container env variables", func() { @@ -808,7 +750,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -830,129 +772,27 @@ var _ = Describe("FeatureStore Controller", func() { } fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) // check online config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // change feast project and reconcile - resourceNew := resource.DeepCopy() - resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} - err = k8sClient.Update(ctx, resourceNew) - Expect(err).NotTo(HaveOccurred()) - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) - }) - - It("Should scale online/offline store service", func() { - By("Reconciling the created resource") - controllerReconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - resource := &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name}) - Expect(err).NotTo(HaveOccurred()) - labelSelector := labels.NewSelector().Add(*req) - listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector} - deployList := appsv1.DeploymentList{} - err = k8sClient.List(ctx, &deployList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) - - svcList := corev1.ServiceList{} - err = k8sClient.List(ctx, &svcList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(svcList.Items).To(HaveLen(3)) - - cmList := corev1.ConfigMapList{} - err = k8sClient.List(ctx, &cmList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) - - feast := services.FeastServices{ - Handler: handler.FeastHandler{ - Client: controllerReconciler.Client, - Context: ctx, - Scheme: controllerReconciler.Scheme, - FeatureStore: resource, - }, - } - - fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) - Expect(err).NotTo(HaveOccurred()) - - // check online config - deploy_online := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_online.Name)) - Expect(deploy_online.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_online.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy_online.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy_online.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // check offline config - deploy_offline := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_offline.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_offline.Name)) - Expect(deploy_offline.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) // change feast project and reconcile - // scale online replicas to 2 resourceNew := resource.DeepCopy() - new_replicas := int32(2) - resourceNew.Spec.Services.OnlineStore.Replicas = &new_replicas - resourceNew.Spec.Services.OfflineStore.Replicas = &new_replicas - + resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -962,21 +802,16 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Replicas).To(Equal(&new_replicas)) - Expect(deploy_offline.Spec.Replicas).To(Equal(&new_replicas)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) It("Should delete k8s objects owned by the FeatureStore CR", func() { @@ -1002,7 +837,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -1021,7 +856,7 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(2)) + Expect(deployList.Items).To(HaveLen(1)) err = k8sClient.List(ctx, &svcList, listOpts) Expect(err).NotTo(HaveOccurred()) @@ -1096,6 +931,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).To(HaveOccurred()) err = k8sClient.Get(ctx, nsName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).NotTo(BeEmpty()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType)).To(BeNil()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType)).To(BeNil()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeFalse()) @@ -1135,7 +971,6 @@ var _ = Describe("FeatureStore Controller", func() { Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType)).To(BeTrue()) - Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).To(Equal(resource.Namespace)) Expect(resource.Status.ServiceHostnames.Registry).ToNot(BeEmpty()) Expect(resource.Status.ServiceHostnames.Registry).To(Equal(referencedRegistry.Status.ServiceHostnames.Registry)) feast := services.FeastServices{ @@ -1147,6 +982,16 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + // check client config cm := &corev1.ConfigMap{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -1235,12 +1080,13 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -1275,7 +1121,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) @@ -1296,10 +1142,9 @@ var _ = Describe("FeatureStore Controller", func() { cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.OfflineStoreFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) Expect(cond).ToNot(BeNil()) @@ -1362,7 +1207,7 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, replicas int32, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -1373,17 +1218,14 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co Services: &feastdevv1alpha1.FeatureStoreServices{ OfflineStore: &feastdevv1alpha1.OfflineStore{}, OnlineStore: &feastdevv1alpha1.OnlineStore{ - StoreServiceConfigs: feastdevv1alpha1.StoreServiceConfigs{ - Replicas: &replicas, - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, - }, + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, }, }, }, @@ -1394,7 +1236,7 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar { for _, e := range envs { - if e.Name == services.FeatureStoreYamlEnvVar { + if e.Name == services.TmpFeatureStoreYamlEnvVar { return &e } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index c191dae3329..0b7fe84d22a 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -27,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -172,16 +171,17 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -225,7 +225,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -237,20 +237,21 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmList.Items).To(HaveLen(1)) - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -259,32 +260,19 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -293,37 +281,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), - Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -332,26 +298,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpsPort, - Scheme: services.HttpsScheme, - Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) + Expect(repoConfigOnline).To(Equal(&testConfig)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) // check client config @@ -366,6 +313,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpsPort, + Scheme: services.HttpsScheme, + Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), + Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -422,25 +381,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check registry deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -454,21 +406,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Path: remoteRegHost, Cert: services.GetTlsPath(services.RegistryFeastType) + "remote.crt", } - offlineConfig.Registry = regRemote - Expect(repoConfigOffline).To(Equal(offlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -478,8 +424,8 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.Registry = regRemote - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOnline).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go index d4b78e2611e..89e22f7be6d 100644 --- a/infra/feast-operator/internal/controller/services/client.go +++ b/infra/feast-operator/internal/controller/services/client.go @@ -32,7 +32,7 @@ func (feast *FeastServices) deployClient() error { func (feast *FeastServices) createClientConfigMap() error { logger := log.FromContext(feast.Handler.Context) cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, cm, controllerutil.MutateFn(func() error { @@ -46,7 +46,7 @@ func (feast *FeastServices) createClientConfigMap() error { } func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { - cm.Labels = feast.getLabels(ClientFeastType) + cm.Labels = feast.getFeastTypeLabels(ClientFeastType) clientYaml, err := feast.getClientFeatureStoreYaml(feast.extractConfigFromSecret) if err != nil { return err @@ -81,7 +81,7 @@ func (feast *FeastServices) setCaConfigMap(cm *corev1.ConfigMap) error { func (feast *FeastServices) initCaConfigMap() *corev1.ConfigMap { cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientCaFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientCaFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) return cm diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 5433e99acfd..675fbd047f8 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -27,36 +27,74 @@ import ( ) // GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service -func (feast *FeastServices) GetServiceFeatureStoreYamlBase64(feastType FeastServiceType) (string, error) { - fsYaml, err := feast.getServiceFeatureStoreYaml(feastType) +func (feast *FeastServices) GetServiceFeatureStoreYamlBase64() (string, error) { + fsYaml, err := feast.getServiceFeatureStoreYaml() if err != nil { return "", err } return base64.StdEncoding.EncodeToString(fsYaml), nil } -func (feast *FeastServices) getServiceFeatureStoreYaml(feastType FeastServiceType) ([]byte, error) { - repoConfig, err := feast.getServiceRepoConfig(feastType) +func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) { + repoConfig, err := feast.getServiceRepoConfig() if err != nil { return nil, err } return yaml.Marshal(repoConfig) } -func (feast *FeastServices) getServiceRepoConfig(feastType FeastServiceType) (RepoConfig, error) { - return getServiceRepoConfig(feastType, feast.Handler.FeatureStore, feast.extractConfigFromSecret) +func (feast *FeastServices) getServiceRepoConfig() (RepoConfig, error) { + return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret) } func getServiceRepoConfig( - feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + repoConfig, err := getBaseServiceRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return repoConfig, err + } + appliedSpec := featureStore.Status.Applied + if appliedSpec.Services != nil { + services := appliedSpec.Services + if services.OfflineStore != nil { + err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if services.OnlineStore != nil { + err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if IsLocalRegistry(featureStore) { + err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + } + + return repoConfig, nil +} - repoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) +func getBaseServiceRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + + appliedSpec := featureStore.Status.Applied + repoConfig := defaultRepoConfig(featureStore) + clientRepoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) if err != nil { return repoConfig, err } + repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig + if isRemoteRegistry(featureStore) { + repoConfig.Registry = clientRepoConfig.Registry + } if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") @@ -75,43 +113,10 @@ func getServiceRepoConfig( repoConfig.AuthzConfig.OidcParameters = oidcServerProperties } - if appliedSpec.Services != nil { - services := appliedSpec.Services - - switch feastType { - case OfflineFeastType: - // Offline server has an `offline_store` section and a remote `registry` - if services.OfflineStore != nil { - err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case OnlineFeastType: - // Online server has an `online_store` section, a remote `registry` and a remote `offline_store` - if services.OnlineStore != nil { - err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case RegistryFeastType: - // Registry server only has a `registry` section - if IsLocalRegistry(featureStore) { - err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - } - } - return repoConfig, nil } func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.Registry = RegistryConfig{} - repoConfig.Registry.Path = DefaultRegistryEphemeralPath registryPersistence := services.Registry.Local.Persistence if registryPersistence != nil { @@ -142,18 +147,10 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr repoConfig.Registry.DBParameters = parametersMap } } - - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OnlineStore = OnlineStoreConfig{} - - repoConfig.OnlineStore.Path = DefaultOnlineStoreEphemeralPath - repoConfig.OnlineStore.Type = OnlineSqliteConfigType onlineStorePersistence := services.OnlineStore.Persistence if onlineStorePersistence != nil { @@ -183,13 +180,11 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret repoConfig.OnlineStore.DBParameters = parametersMap } } - return nil } func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OfflineStore.Type = OfflineFilePersistenceDaskConfigType + repoConfig.OfflineStore = defaultOfflineStoreConfig offlineStorePersistence := services.OfflineStore.Persistence if offlineStorePersistence != nil { @@ -218,9 +213,6 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre repoConfig.OfflineStore.DBParameters = parametersMap } } - - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } @@ -237,10 +229,9 @@ func getClientRepoConfig( secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { status := featureStore.Status appliedServices := status.Applied.Services - clientRepoConfig := RepoConfig{ - Project: status.Applied.FeastProject, - Provider: LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + clientRepoConfig, err := getRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return clientRepoConfig, err } if len(status.ServiceHostnames.OfflineStore) > 0 { clientRepoConfig.OfflineStore = OfflineStoreConfig{ @@ -277,23 +268,27 @@ func getClientRepoConfig( } } - if status.Applied.AuthzConfig == nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ - Type: NoAuthAuthType, - } - } else { + return clientRepoConfig, nil +} + +func getRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + status := featureStore.Status + repoConfig := initRepoConfig(status.Applied.FeastProject) + if status.Applied.AuthzConfig != nil { if status.Applied.AuthzConfig.KubernetesAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: KubernetesAuthType, } } else if status.Applied.AuthzConfig.OidcAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: OidcAuthType, } propertiesMap, err := secretExtractionFunc("", status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") if err != nil { - return clientRepoConfig, err + return repoConfig, err } oidcClientProperties := map[string]interface{}{} @@ -301,13 +296,13 @@ func getClientRepoConfig( if val, exists := propertiesMap[string(oidcClientProperty)]; exists { oidcClientProperties[string(oidcClientProperty)] = val } else { - return clientRepoConfig, missingOidcSecretProperty(oidcClientProperty) + return repoConfig, missingOidcSecretProperty(oidcClientProperty) } } - clientRepoConfig.AuthzConfig.OidcParameters = oidcClientProperties + repoConfig.AuthzConfig.OidcParameters = oidcClientProperties } } - return clientRepoConfig, nil + return repoConfig, nil } func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) string { @@ -372,3 +367,49 @@ func mergeStructWithDBParametersMap(parametersMap *map[string]interface{}, s int return nil } + +func (feast *FeastServices) GetDefaultRepoConfig() RepoConfig { + return defaultRepoConfig(feast.Handler.FeatureStore) +} + +func defaultRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig { + repoConfig := initRepoConfig(featureStore.Status.Applied.FeastProject) + repoConfig.OnlineStore = defaultOnlineStoreConfig(featureStore) + repoConfig.Registry = defaultRegistryConfig(featureStore) + return repoConfig +} + +func (feast *FeastServices) GetInitRepoConfig() RepoConfig { + return initRepoConfig(feast.Handler.FeatureStore.Status.Applied.FeastProject) +} + +func initRepoConfig(feastProject string) RepoConfig { + return RepoConfig{ + Project: feastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + AuthzConfig: defaultAuthzConfig, + } +} + +func defaultOnlineStoreConfig(featureStore *feastdevv1alpha1.FeatureStore) OnlineStoreConfig { + return OnlineStoreConfig{ + Type: OnlineSqliteConfigType, + Path: defaultOnlineStorePath(featureStore), + } +} + +func defaultRegistryConfig(featureStore *feastdevv1alpha1.FeatureStore) RegistryConfig { + return RegistryConfig{ + RegistryType: RegistryFileConfigType, + Path: defaultRegistryPath(featureStore), + } +} + +var defaultOfflineStoreConfig = OfflineStoreConfig{ + Type: OfflineFilePersistenceDaskConfigType, +} + +var defaultAuthzConfig = AuthzConfig{ + Type: NoAuthAuthType, +} diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 7f017f4d102..42525700a4c 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -32,35 +32,26 @@ var projectName = "test-project" var _ = Describe("Repo Config", func() { Context("When creating the RepoConfig of a FeatureStore", func() { - It("should successfully create the repo configs", func() { By("Having the minimal created resource") featureStore := minimalFeatureStore() ApplyDefaultsToStatus(featureStore) - var repoConfig RepoConfig - repoConfig, err := getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig := RegistryConfig{ RegistryType: "file", - Path: DefaultRegistryEphemeralPath, + Path: EphemeralPath + "/" + DefaultRegistryPath, + } + expectedOnlineConfig := OnlineStoreConfig{ + Type: "sqlite", + Path: EphemeralPath + "/" + DefaultOnlineStorePath, } + + var repoConfig RepoConfig + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the local registry resource") @@ -77,29 +68,17 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: "file", Path: "file.db", } + + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the remote registry resource") @@ -108,33 +87,18 @@ var _ = Describe("Repo Config", func() { Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ FeastRef: &feastdevv1alpha1.FeatureStoreRef{ - Name: "registry", - Namespace: "remoteNS", + Name: "registry", }, }, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) By("Having the all the file services") featureStore = minimalFeatureStore() @@ -164,36 +128,24 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + expectedOfflineConfig := OfflineStoreConfig{ Type: "duckdb", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig := OnlineStoreConfig{ + expectedRegistryConfig = RegistryConfig{ + RegistryType: "file", + Path: "/data/registry.db", + } + expectedOnlineConfig = OnlineStoreConfig{ Type: "sqlite", Path: "/data/online.db", } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: "/data/registry.db", - } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having kubernetes authorization") @@ -202,56 +154,24 @@ var _ = Describe("Repo Config", func() { KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{}, } featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{ - Persistence: &feastdevv1alpha1.OfflineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{}, - }, - }, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - Persistence: &feastdevv1alpha1.OnlineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OnlineStoreFilePersistence{}, - }, - }, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, Registry: &feastdevv1alpha1.Registry{ - Local: &feastdevv1alpha1.LocalRegistryConfig{ - Persistence: &feastdevv1alpha1.RegistryPersistence{ - FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{}, - }, - }, + Local: &feastdevv1alpha1.LocalRegistryConfig{}, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) + expectedOfflineConfig = OfflineStoreConfig{ Type: "dask", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) By("Having oidc authorization") featureStore.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{ @@ -269,46 +189,15 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - expectedOfflineConfig = OfflineStoreConfig{ - Type: "dask", - } Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) repoConfig, err = getClientRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) @@ -359,7 +248,7 @@ var _ = Describe("Repo Config", func() { featureStore.Spec.Services.OfflineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.OnlineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.Registry.Local.Persistence.FilePersistence = nil - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) newMap := CopyMap(parameterMap) port := parameterMap["port"].(int) @@ -369,29 +258,16 @@ var _ = Describe("Repo Config", func() { Port: port, DBParameters: newMap, } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - newMap = CopyMap(parameterMap) expectedOnlineConfig = OnlineStoreConfig{ Type: OnlineDBPersistenceSnowflakeConfigType, - DBParameters: newMap, + DBParameters: CopyMap(parameterMap), } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: RegistryDBPersistenceSnowflakeConfigType, DBParameters: parameterMap, } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) }) }) @@ -413,13 +289,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err := getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err := getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -440,13 +316,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): "client-id", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -455,17 +331,8 @@ var _ = Describe("Repo Config", func() { }) }) -func emptyOnlineStoreConfig() OnlineStoreConfig { - return OnlineStoreConfig{} -} - -func emptyOfflineStoreConfig() OfflineStoreConfig { - return OfflineStoreConfig{} -} - -func emptyRegistryConfig() RegistryConfig { - return RegistryConfig{} -} +var emptyOfflineStoreConfig = OfflineStoreConfig{} +var emptyRegistryConfig = RegistryConfig{} func minimalFeatureStore() *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 232e3e58743..32cf91d09ba 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -106,7 +106,12 @@ func (feast *FeastServices) Deploy() error { return err } } - + if err := feast.createServiceAccount(); err != nil { + return err + } + if err := feast.createDeployment(); err != nil { + return err + } if err := feast.deployClient(); err != nil { return err } @@ -194,12 +199,6 @@ func (feast *FeastServices) deployFeastServiceByType(feastType FeastServiceType) if err := feast.createService(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createServiceAccount(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } - if err := feast.createDeployment(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } return feast.setFeastServiceCondition(nil, feastType) } @@ -207,12 +206,6 @@ func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { return err } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastDeploy(feastType)); err != nil { - return err - } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSA(feastType)); err != nil { - return err - } if err := feast.Handler.DeleteOwnedFeastObj(feast.initPVC(feastType)); err != nil { return err } @@ -233,11 +226,11 @@ func (feast *FeastServices) createService(feastType FeastServiceType) error { return nil } -func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) error { +func (feast *FeastServices) createServiceAccount() error { logger := log.FromContext(feast.Handler.Context) - sa := feast.initFeastSA(feastType) + sa := feast.initFeastSA() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, controllerutil.MutateFn(func() error { - return feast.setServiceAccount(sa, feastType) + return feast.setServiceAccount(sa) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -246,11 +239,11 @@ func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) err return nil } -func (feast *FeastServices) createDeployment(feastType FeastServiceType) error { +func (feast *FeastServices) createDeployment() error { logger := log.FromContext(feast.Handler.Context) - deploy := feast.initFeastDeploy(feastType) + deploy := feast.initFeastDeploy() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, deploy, controllerutil.MutateFn(func() error { - return feast.setDeployment(deploy, feastType) + return feast.setDeployment(deploy) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -280,78 +273,122 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1alpha1.PvcCreate, fea return nil } -func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType FeastServiceType) error { - fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64(feastType) - if err != nil { - return err - } - deploy.Labels = feast.getLabels(feastType) - sa := feast.initFeastSA(feastType) - tls := feast.getTlsConfigs(feastType) - serviceConfigs := feast.getServiceConfigs(feastType) - defaultServiceConfigs := serviceConfigs.DefaultConfigs - probeHandler := getProbeHandler(feastType, tls) - +func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { + deploy.Labels = feast.getLabels() deploy.Spec = appsv1.DeploymentSpec{ - Replicas: feast.getServiceReplicas(feastType), + Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deploy.GetLabels(), }, Spec: corev1.PodSpec{ - ServiceAccountName: sa.Name, - Containers: []corev1.Container{ - { - Name: string(feastType), - Image: *defaultServiceConfigs.Image, - Command: feast.getContainerCommand(feastType), - Ports: []corev1.ContainerPort{ - { - Name: string(feastType), - ContainerPort: getTargetPort(feastType, tls), - Protocol: corev1.ProtocolTCP, - }, - }, - Env: []corev1.EnvVar{ - { - Name: FeatureStoreYamlEnvVar, - Value: fsYamlB64, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 30, - PeriodSeconds: 30, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 20, - PeriodSeconds: 30, - }, - }, - }, + ServiceAccountName: feast.initFeastSA().Name, }, }, } + if err := feast.setPod(&deploy.Spec.Template.Spec); err != nil { + return err + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +} - // configs are applied here - podSpec := &deploy.Spec.Template.Spec - applyOptionalContainerConfigs(&podSpec.Containers[0], serviceConfigs.OptionalConfigs) - feast.mountTlsConfig(feastType, podSpec) - if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { - mountPvcConfig(podSpec, pvcConfig, deploy.Name) +func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { + if err := feast.setContainers(podSpec); err != nil { + return err } + feast.setRegistryClientInitContainer(podSpec) + feast.mountTlsConfigs(podSpec) + feast.mountPvcConfigs(podSpec) + feast.mountEmptyDirVolumes(podSpec) - switch feastType { - case OfflineFeastType: - feast.registryClientPodConfigs(podSpec) - case OnlineFeastType: - feast.registryClientPodConfigs(podSpec) - feast.offlineClientPodConfigs(podSpec) + return nil +} + +func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() + if err != nil { + return err + } + feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject + workingDir := getOfflineMountPath(feast.Handler.FeatureStore) + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "feast-init", + Image: DefaultImage, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + + feastProject + " ] || feast init " + feastProject + ";\necho $" + + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastProject + + "/feature_repo/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + WorkingDir: workingDir, + }) + if feast.isLocalRegistry() { + feast.setContainer(&podSpec.Containers, RegistryFeastType, fsYamlB64) } + if feast.isOfflinStore() { + feast.setContainer(&podSpec.Containers, OfflineFeastType, fsYamlB64) + } + if feast.isOnlinStore() { + feast.setContainer(&podSpec.Containers, OnlineFeastType, fsYamlB64) + } + return nil +} - return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastType FeastServiceType, fsYamlB64 string) { + tls := feast.getTlsConfigs(feastType) + serviceConfigs := feast.getServiceConfigs(feastType) + defaultServiceConfigs := serviceConfigs.DefaultConfigs + probeHandler := getProbeHandler(feastType, tls) + container := &corev1.Container{ + Name: string(feastType), + Image: *defaultServiceConfigs.Image, + WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "/feature_repo", + Command: feast.getContainerCommand(feastType), + Ports: []corev1.ContainerPort{ + { + Name: string(feastType), + ContainerPort: getTargetPort(feastType, tls), + Protocol: corev1.ProtocolTCP, + }, + }, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + /* + { + Name: mlpConfigVar, + Value: DefaultMlpConfigPath, + }, + */ + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 3, + FailureThreshold: 40, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 20, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 10, + }, + } + applyOptionalContainerConfigs(container, serviceConfigs.OptionalConfigs) + *containers = append(*containers, *container) } func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { @@ -380,38 +417,28 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st return feastCommand } -func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { - feast.mountTlsConfig(OfflineFeastType, podSpec) -} - -func (feast *FeastServices) registryClientPodConfigs(podSpec *corev1.PodSpec) { - feast.setRegistryClientInitContainer(podSpec) - feast.mountRegistryClientTls(podSpec) -} - func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry - if len(hostname) > 0 { + // add grpc init container if remote registry reference (feastRef) is configured + if len(hostname) > 0 && feast.IsRemoteRefRegistry() { grpcurlFlag := "-plaintext" hostSplit := strings.Split(hostname, ":") if len(hostSplit) > 1 && hostSplit[1] == "443" { grpcurlFlag = "-insecure" } - podSpec.InitContainers = []corev1.Container{ - { - Name: "init-registry", - Image: "fullstorydev/grpcurl:v1.9.1-alpine", - Command: []string{ - "sh", "-c", - "until grpcurl " + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", - }, + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "init-registry", + Image: "fullstorydev/grpcurl:v1.9.1-alpine", + Command: []string{ + "sh", "-c", + "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", }, - } + }) } } - func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { - svc.Labels = feast.getLabels(feastType) + svc.Labels = feast.getFeastTypeLabels(feastType) if feast.isOpenShiftTls(feastType) { svc.Annotations = map[string]string{ "service.beta.openshift.io/serving-cert-secret-name": svc.Name + tlsNameSuffix, @@ -426,7 +453,7 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi scheme = HttpsScheme } svc.Spec = corev1.ServiceSpec{ - Selector: svc.GetLabels(), + Selector: feast.getLabels(), Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { @@ -441,8 +468,8 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi return controllerutil.SetControllerReference(feast.Handler.FeatureStore, svc, feast.Handler.Scheme) } -func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount, feastType FeastServiceType) error { - sa.Labels = feast.getLabels(feastType) +func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount) error { + sa.Labels = feast.getLabels() return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) } @@ -478,21 +505,6 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } -func (feast *FeastServices) getServiceReplicas(feastType FeastServiceType) *int32 { - appliedServices := feast.Handler.FeatureStore.Status.Applied.Services - switch feastType { - case OfflineFeastType: - if feast.isOfflinStore() { - return appliedServices.OfflineStore.Replicas - } - case OnlineFeastType: - if feast.isOnlinStore() { - return appliedServices.OnlineStore.Replicas - } - } - return &DefaultReplicas -} - func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { services := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { @@ -512,8 +524,13 @@ func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *stri return nil } -// GetObjectMeta returns the feast k8s object metadata -func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMeta() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: GetFeastName(feast.Handler.FeatureStore), Namespace: feast.Handler.FeatureStore.Namespace} +} + +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMetaType(feastType FeastServiceType) metav1.ObjectMeta { return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.Handler.FeatureStore.Namespace} } @@ -530,10 +547,15 @@ func GetFeastName(featureStore *feastdevv1alpha1.FeatureStore) string { return handler.FeastPrefix + featureStore.Name } -func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]string { +func (feast *FeastServices) getFeastTypeLabels(feastType FeastServiceType) map[string]string { + labels := feast.getLabels() + labels[ServiceTypeLabelKey] = string(feastType) + return labels +} + +func (feast *FeastServices) getLabels() map[string]string { return map[string]string{ - NameLabelKey: feast.Handler.FeatureStore.Name, - ServiceTypeLabelKey: string(feastType), + NameLabelKey: feast.Handler.FeatureStore.Name, } } @@ -541,17 +563,17 @@ func (feast *FeastServices) setServiceHostnames() error { feast.Handler.FeatureStore.Status.ServiceHostnames = feastdevv1alpha1.ServiceHostnames{} domain := svcDomain + ":" if feast.isOfflinStore() { - objMeta := feast.GetObjectMeta(OfflineFeastType) + objMeta := feast.initFeastSvc(OfflineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS) } if feast.isOnlinStore() { - objMeta := feast.GetObjectMeta(OnlineFeastType) + objMeta := feast.initFeastSvc(OnlineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OnlineStore.TLS) } if feast.isLocalRegistry() { - objMeta := feast.GetObjectMeta(RegistryFeastType) + objMeta := feast.initFeastSvc(RegistryFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.Registry.Local.TLS) } else if feast.isRemoteRegistry() { @@ -598,10 +620,6 @@ func (feast *FeastServices) setRemoteRegistryURL() error { func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, error) { if feast.IsRemoteRefRegistry() { feastRemoteRef := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef - // default to FeatureStore namespace if not set - if len(feastRemoteRef.Namespace) == 0 { - feastRemoteRef.Namespace = feast.Handler.FeatureStore.Namespace - } nsName := types.NamespacedName{Name: feastRemoteRef.Name, Namespace: feastRemoteRef.Namespace} crNsName := client.ObjectKeyFromObject(feast.Handler.FeatureStore) if nsName == crNsName { @@ -635,19 +653,13 @@ func (feast *FeastServices) isRemoteRegistry() bool { } func (feast *FeastServices) IsRemoteRefRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.FeastRef != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef != nil } func (feast *FeastServices) isRemoteHostnameRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.Hostname != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.Hostname != nil } func (feast *FeastServices) isOfflinStore() bool { @@ -660,9 +672,9 @@ func (feast *FeastServices) isOnlinStore() bool { return appliedServices != nil && appliedServices.OnlineStore != nil } -func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1.Deployment { +func (feast *FeastServices) initFeastDeploy() *appsv1.Deployment { deploy := &appsv1.Deployment{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } deploy.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment")) return deploy @@ -670,15 +682,15 @@ func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1. func (feast *FeastServices) initFeastSvc(feastType FeastServiceType) *corev1.Service { svc := &corev1.Service{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } svc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) return svc } -func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.ServiceAccount { +func (feast *FeastServices) initFeastSA() *corev1.ServiceAccount { sa := &corev1.ServiceAccount{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) return sa @@ -686,7 +698,7 @@ func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.Serv func (feast *FeastServices) initPVC(feastType FeastServiceType) *corev1.PersistentVolumeClaim { pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } pvc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim")) return pvc @@ -704,28 +716,73 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs } } -func mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, deployName string) { +func (feast *FeastServices) mountPvcConfigs(podSpec *corev1.PodSpec) { + for _, feastType := range feastServerTypes { + if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { + feast.mountPvcConfig(podSpec, pvcConfig, feastType) + } + } +} + +func (feast *FeastServices) mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { if podSpec != nil && pvcConfig != nil { - container := &podSpec.Containers[0] - var pvcName string - if pvcConfig.Create != nil { - pvcName = deployName - } else { + volName := feast.initPVC(feastType).Name + pvcName := volName + if pvcConfig.Ref != nil { pvcName = pvcConfig.Ref.Name } - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: pvcName, + Name: volName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcName, }, }, }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: pvcName, - MountPath: pvcConfig.MountPath, + if feastType == OfflineFeastType { + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } +} + +func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { + if shouldMountEmptyDir(feast.Handler.FeatureStore) { + mountEmptyDirVolume(podSpec) + } +} + +func mountEmptyDirVolume(podSpec *corev1.PodSpec) { + if podSpec != nil { + volName := strings.TrimPrefix(EphemeralPath, "/") + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, }) + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index b9e1a9d9d75..882839a429f 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -25,13 +25,12 @@ import ( ) const ( - FeatureStoreYamlEnvVar = "FEATURE_STORE_YAML_BASE64" - FeatureStoreYamlCmKey = "feature_store.yaml" - DefaultRegistryEphemeralPath = "/tmp/registry.db" - DefaultRegistryPvcPath = "registry.db" - DefaultOnlineStoreEphemeralPath = "/tmp/online_store.db" - DefaultOnlineStorePvcPath = "online_store.db" - svcDomain = ".svc.cluster.local" + TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + FeatureStoreYamlCmKey = "feature_store.yaml" + EphemeralPath = "/feast-data" + DefaultRegistryPath = "registry.db" + DefaultOnlineStorePath = "online_store.db" + svcDomain = ".svc.cluster.local" HttpPort = 80 HttpsPort = 443 @@ -164,6 +163,13 @@ var ( OidcClientProperties = []OidcPropertyType{OidcClientSecret, OidcUsername, OidcPassword} ) +// feast server types, not the client types +var feastServerTypes = []FeastServiceType{ + RegistryFeastType, + OfflineFeastType, + OnlineFeastType, +} + // AuthzType defines the authorization type type AuthzType string diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index a52cc707eb3..6dcca7edea1 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -165,12 +165,19 @@ func (feast *FeastServices) mountRegistryClientTls(podSpec *corev1.PodSpec) { if feast.localRegistryTls() { feast.mountTlsConfig(RegistryFeastType, podSpec) } else if feast.remoteRegistryTls() { - mountTlsRemoteRegistryConfig(RegistryFeastType, podSpec, + mountTlsRemoteRegistryConfig(podSpec, feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.TLS) } } } +func (feast *FeastServices) mountTlsConfigs(podSpec *corev1.PodSpec) { + // how deal w/ client deployment tls mounts when the time comes? new function? + feast.mountRegistryClientTls(podSpec) + feast.mountTlsConfig(OfflineFeastType, podSpec) + feast.mountTlsConfig(OnlineFeastType, podSpec) +} + func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec *corev1.PodSpec) { tls := feast.getTlsConfigs(feastType) if tls.IsTLS() && podSpec != nil { @@ -183,18 +190,19 @@ func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec * }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + if i, container := getContainerByType(feastType, podSpec.Containers); container != nil { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } } } -func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { +func mountTlsRemoteRegistryConfig(podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { if tls != nil { - volName := string(feastType) + tlsNameSuffix + volName := string(RegistryFeastType) + tlsNameSuffix podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: volName, VolumeSource: corev1.VolumeSource{ @@ -203,12 +211,13 @@ func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.Po }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(RegistryFeastType), + ReadOnly: true, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 17d23dcf72a..522eb2265b5 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -135,23 +135,15 @@ var _ = Describe("TLS Config", func() { Expect(openshiftTls).To(BeTrue()) // check k8s deployment objects - offlineDeploy := feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) + feastDeploy := feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - onlineDeploy := feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) - Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[1].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[2].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(4)) // registry service w/ tls and in an openshift cluster feast.Handler.FeatureStore = minimalFeatureStore() @@ -258,22 +250,19 @@ var _ = Describe("TLS Config", func() { Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) // check k8s deployment objects - offlineDeploy = feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) - Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - onlineDeploy = feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) + feastDeploy = feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).To(ContainElements(ContainSubstring("--key"))) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 92ee2b5752e..d0ca94ff865 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -32,20 +32,22 @@ func isRemoteRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { } func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastServiceType) (*feastdevv1alpha1.PvcConfig, bool) { + var pvcConfig *feastdevv1alpha1.PvcConfig services := featureStore.Status.Applied.Services - var pvcConfig *feastdevv1alpha1.PvcConfig = nil - switch feastType { - case OnlineFeastType: - if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig - } - case OfflineFeastType: - if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig - } - case RegistryFeastType: - if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { - pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + if services != nil { + switch feastType { + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig + } + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig + } + case RegistryFeastType: + if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { + pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + } } } return pvcConfig, pvcConfig != nil @@ -58,6 +60,20 @@ func shouldCreatePvc(featureStore *feastdevv1alpha1.FeatureStore, feastType Feas return nil, false } +func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { + _, ok := hasPvcConfig(featureStore, OfflineFeastType) + return !ok +} + +func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if featureStore.Status.Applied.Services != nil { + if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { + return pvcConfig.MountPath + } + } + return EphemeralPath +} + func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { cr.Status.FeastVersion = feastversion.FeastVersion applied := cr.Spec.DeepCopy() @@ -87,7 +103,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.Registry.Local.Persistence.FilePersistence.Path) == 0 { - services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(services.Registry.Local.Persistence.FilePersistence) + services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(cr) } if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { @@ -97,7 +113,10 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } setServiceDefaultConfigs(&services.Registry.Local.ServiceConfigs.DefaultConfigs) + } else if services.Registry.Remote.FeastRef != nil && len(services.Registry.Remote.FeastRef.Namespace) == 0 { + services.Registry.Remote.FeastRef.Namespace = cr.Namespace } + if services.OfflineStore != nil { if services.OfflineStore.Persistence == nil { services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{} @@ -118,7 +137,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setStoreServiceDefaultConfigs(&services.OfflineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) } if services.OnlineStore != nil { @@ -132,7 +151,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(services.OnlineStore.Persistence.FilePersistence) + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) } if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { @@ -141,7 +160,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setStoreServiceDefaultConfigs(&services.OnlineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) } // overwrite status.applied with every reconcile applied.DeepCopyInto(&cr.Status.Applied) @@ -153,13 +172,6 @@ func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { } } -func setStoreServiceDefaultConfigs(storeServiceConfigs *feastdevv1alpha1.StoreServiceConfigs) { - if storeServiceConfigs.Replicas == nil { - storeServiceConfigs.Replicas = &DefaultReplicas - } - setServiceDefaultConfigs(&storeServiceConfigs.ServiceConfigs.DefaultConfigs) -} - func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1alpha1.ValidOfflineStoreFilePersistenceTypes, value) { return nil @@ -194,18 +206,20 @@ func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceTy } } -func defaultOnlineStorePath(persistence *feastdevv1alpha1.OnlineStoreFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultOnlineStoreEphemeralPath +func defaultOnlineStorePath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, OnlineFeastType); ok { + return DefaultOnlineStorePath } - return DefaultOnlineStorePvcPath + // if online pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultOnlineStorePath } -func defaultRegistryPath(persistence *feastdevv1alpha1.RegistryFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultRegistryEphemeralPath +func defaultRegistryPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, RegistryFeastType); ok { + return DefaultRegistryPath } - return DefaultRegistryPvcPath + // if registry pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultRegistryPath } func checkOfflineStoreDBStorePersistenceType(value string) error { @@ -357,3 +371,69 @@ func envOverride(dst, src []corev1.EnvVar) []corev1.EnvVar { } return dst } + +func GetRegistryContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(RegistryFeastType, containers) + return container +} + +func GetOfflineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OfflineFeastType, containers) + return container +} + +func GetOnlineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OnlineFeastType, containers) + return container +} + +func getContainerByType(feastType FeastServiceType, containers []corev1.Container) (int, *corev1.Container) { + for i, c := range containers { + if c.Name == string(feastType) { + return i, &c + } + } + return -1, nil +} + +func GetRegistryVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(RegistryFeastType, featureStore, volumes) +} + +func GetOnlineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OnlineFeastType, featureStore, volumes) +} + +func GetOfflineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OfflineFeastType, featureStore, volumes) +} + +func getVolumeByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + for _, v := range volumes { + if v.Name == GetFeastServiceName(featureStore, feastType) { + return &v + } + } + return nil +} + +func GetRegistryVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(RegistryFeastType, featureStore, volumeMounts) +} + +func GetOnlineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OnlineFeastType, featureStore, volumeMounts) +} + +func GetOfflineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OfflineFeastType, featureStore, volumeMounts) +} + +func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + for _, vm := range volumeMounts { + if vm.Name == GetFeastServiceName(featureStore, feastType) { + return &vm + } + } + return nil +} diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index fdf58d8f3b7..bab57a3c006 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -27,9 +27,12 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/test/utils" ) -const feastControllerNamespace = "feast-operator-system" -const timeout = 2 * time.Minute -const controllerDeploymentName = "feast-operator-controller-manager" +const ( + feastControllerNamespace = "feast-operator-system" + timeout = 2 * time.Minute + controllerDeploymentName = "feast-operator-controller-manager" + feastPrefix = "feast-" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { @@ -159,15 +162,18 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", featureStoreName, err)) - k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName), - fmt.Sprintf("feast-%s-offline", featureStoreName), + feastResourceName := feastPrefix + featureStoreName + k8sResourceNames := []string{feastResourceName} + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", } if !hasRemoteRegistry { - k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName)) + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") } - for _, deploymentName := range k8ResourceNames { + for _, deploymentName := range k8sResourceNames { By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) Expect(err).To(BeNil(), fmt.Sprintf( @@ -178,7 +184,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st } By("Check if the feast client - kubernetes config map exists.") - configMapName := fmt.Sprintf("feast-%s-client", featureStoreName) + configMapName := feastResourceName + "-client" err = checkIfConfigMapExists(namespace, configMapName) Expect(err).To(BeNil(), fmt.Sprintf( "config map %s is not available but expected to be available. \nError: %v\n", @@ -186,7 +192,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st )) fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) - for _, serviceAccountName := range k8ResourceNames { + for _, serviceAccountName := range k8sResourceNames { By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) err = checkIfServiceAccountExists(namespace, serviceAccountName) Expect(err).To(BeNil(), fmt.Sprintf( @@ -196,7 +202,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) } - for _, serviceName := range k8ResourceNames { + for _, serviceName := range feastK8sResourceNames { By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) err = checkIfKubernetesServiceExists(namespace, serviceName) Expect(err).To(BeNil(), fmt.Sprintf( diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index d92f719fb97..017690a0ec9 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -192,9 +192,5 @@ func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (boo hasValidFeastRef := registryConfig.Remote.FeastRef != nil && registryConfig.Remote.FeastRef.Name != "" - if hasHostname || hasValidFeastRef { - return true, nil - } - - return false, nil + return hasHostname || hasValidFeastRef, nil } From b539ebaad5ec2c1a199fe08ceccd206754ce82f0 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Fri, 20 Dec 2024 15:21:54 -0600 Subject: [PATCH 51/90] feat: Add duckdb extra to multicloud release image (#4862) add duckdb extra to multicloud release image Signed-off-by: Tommy Hughes --- sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index c1da48f55d0..b4d7b5e3e9c 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11-slim-bullseye RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s]" +RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb]" RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ From dbc92070c8ef6b9e4e53d89ec03090bf30bd0f60 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Fri, 20 Dec 2024 21:07:54 -0500 Subject: [PATCH 52/90] feat: Snyk vulnerability issues fix. (#4867) * Update README.md Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * chore: Update quickstart.md Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Feature/lrangine master (#6) * Snyk scan vulnerability fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Reverting the grpc version so hoping that it will fix the java integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Upgrading the grpc version as it didn't fix the problem Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * adding grpc-api libraries as dependency to solve some of the class not found exceptions with the grpc upgrades. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Fix for 2 vulnerabilities (#3) * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-5710356 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating the requirements files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating the requirements files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Changing the python httpx package to 0.27.2 because after 0.28.0 version is giving errors related to proxies which is removed. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Security upgrade io.grpc:grpc-services from 1.53.0 to 1.63.0 (#4) * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Fix for 1 vulnerabilities (#5) Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-5710356 --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * trying to fix some vulnerabilities in the requirements.txt files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Co-authored-by: snyk-bot Co-authored-by: Francisco Arceo Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Feature/lrangine master (#7) * feat: Loading the CA trusted store certificate into Feast to verify the public certificate. (#4852) * Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the lint error. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Trying to fix the online store test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Formatted the python to fix lint errors. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the unit test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Fixing the unit test cases. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * removing unnecessary cli args. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Now configuring the SSL ca store configurations on the feast client side rather than on the server side. And also fixing the integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Renamed the remote registry is_tls_mode variable to is_tls. Changed the offline store TLS setting decision from cert to scheme. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Adding the existing trust store certificates to the newly created trust store. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * combining the default system ca store with the custom one to fix the integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Final clean up and adding documentation. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Incorporating the code review comments from Francisco. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: Updated python-helm-demo example to use MinIO instead of GS (#4691) * Updated python-helm-demo example to use MinIO instead of GS Signed-off-by: Daniele Martinoli * Update examples/python-helm-demo/README.md Co-authored-by: Francisco Arceo Signed-off-by: Daniele Martinoli * Adding explicit wait to container to validate CI failures Signed-off-by: Daniele Martinoli * restored original conftest Signed-off-by: Daniele Martinoli --------- Signed-off-by: Daniele Martinoli Co-authored-by: Francisco Arceo Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: Fixing some of the warnings with the github actions (#4763) Fixing some of the warnings with the github actions, most of them related to deprecated actions or libraries. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Update README.md Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Snyk scan vulnerability fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Reverting the grpc version so hoping that it will fix the java integration tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Upgrading the grpc version as it didn't fix the problem Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * adding grpc-api libraries as dependency to solve some of the class not found exceptions with the grpc upgrades. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Fix for 2 vulnerabilities (#3) * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-5710356 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating the requirements files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating the requirements files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Changing the python httpx package to 0.27.2 because after 0.28.0 version is giving errors related to proxies which is removed. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Security upgrade io.grpc:grpc-services from 1.53.0 to 1.63.0 (#4) * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * [Snyk] Fix for 1 vulnerabilities (#5) Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * chore: Update quickstart.md * fix: java/serving/pom.xml & java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEOAUTHCLIENT-2807808 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055227 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-8055228 - https://snyk.io/vuln/SNYK-JAVA-ORGYAML-3152153 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3167772 - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHETHRIFT-1074898 - https://snyk.io/vuln/SNYK-JAVA-IONETTY-6483812 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLECODEGSON-1730327 - https://snyk.io/vuln/SNYK-JAVA-COMSQUAREUPOKHTTP3-2958044 - https://snyk.io/vuln/SNYK-JAVA-IOGRPC-571957 - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEPROTOBUF-3040284 - https://snyk.io/vuln/SNYK-JAVA-JUNIT-1017047 * fix: sdk/python/feast/ui/package.json & sdk/python/feast/ui/yarn.lock to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-TRIM-1017038 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * fix: java/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-5710356 --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * trying to fix some vulnerabilities in the requirements.txt files. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating the lettuce-core to fix the snyk vulnerability. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Signed-off-by: Daniele Martinoli Co-authored-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> Co-authored-by: Francisco Arceo Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * updating netty library Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * updating netty library Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Updating aws java sdk libraries. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Adding verbose logs to debug the sudden failure of tests without any error messages. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * Adding verbose logs to debug the sudden failure of tests without any error messages. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Signed-off-by: Daniele Martinoli Co-authored-by: Francisco Arceo Co-authored-by: snyk-bot Co-authored-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> Co-authored-by: Francisco Arceo --- Makefile | 4 +- java/datatypes/pom.xml | 5 + java/pom.xml | 16 +-- java/serving-client/pom.xml | 5 + java/serving/pom.xml | 17 ++- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 110 ++---------------- .../requirements/py3.10-ci-requirements.txt | 108 ++++++++--------- .../requirements/py3.10-requirements.txt | 34 +++--- .../requirements/py3.11-ci-requirements.txt | 107 +++++++++-------- .../requirements/py3.11-requirements.txt | 33 +++--- .../requirements/py3.9-ci-requirements.txt | 103 ++++++++-------- .../requirements/py3.9-requirements.txt | 30 ++--- setup.py | 6 +- 14 files changed, 258 insertions(+), 322 deletions(-) diff --git a/Makefile b/Makefile index d7374d347c8..de2ee568b68 100644 --- a/Makefile +++ b/Makefile @@ -96,14 +96,14 @@ test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ sdk/python/tests test-python-integration-local: FEAST_IS_LOCAL_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ - python -m pytest -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ sdk/python/tests diff --git a/java/datatypes/pom.xml b/java/datatypes/pom.xml index b0ba049c575..967262d0e01 100644 --- a/java/datatypes/pom.xml +++ b/java/datatypes/pom.xml @@ -118,6 +118,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + javax.annotation javax.annotation-api diff --git a/java/pom.xml b/java/pom.xml index 6b18588923a..82c0a00ba22 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -41,9 +41,9 @@ UTF-8 UTF-8 - 1.30.2 + 1.63.0 3.12.2 - 3.16.1 + 3.25.5 1.111.1 0.8.0 1.9.10 @@ -61,15 +61,15 @@ 1.5.24 3.14.7 3.10 - 2.14.0 + 2.15.0 2.3.1 1.3.2 2.0.1.Final 0.21.0 1.6.6 - 30.1-jre + 32.0.0-jre 3.4.34 - 4.1.101.Final + 4.1.96.Final src/main/java/**/BatchLoadsWithResult.java - + @@ -365,7 +365,7 @@ [11.0,) - + @@ -376,7 +376,7 @@ - + diff --git a/java/serving-client/pom.xml b/java/serving-client/pom.xml index 7b8838a009c..dc611b4a76e 100644 --- a/java/serving-client/pom.xml +++ b/java/serving-client/pom.xml @@ -50,6 +50,11 @@ grpc-testing ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + com.google.protobuf protobuf-java-util diff --git a/java/serving/pom.xml b/java/serving/pom.xml index ca7f8a73b5f..1be4da1b622 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -126,7 +126,7 @@ com.azure azure-storage-blob - 12.25.2 + 12.26.1 com.azure @@ -164,6 +164,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + io.grpc grpc-netty-shaded @@ -192,7 +197,7 @@ io.jaegertracing jaeger-client - 1.3.2 + 1.8.1 io.opentracing @@ -240,7 +245,7 @@ com.google.cloud google-cloud-storage - 1.118.0 + 2.43.1 @@ -253,13 +258,13 @@ com.amazonaws aws-java-sdk-s3 - 1.12.261 + 1.12.546 com.amazonaws aws-java-sdk-sts - 1.12.476 + 1.12.546 @@ -378,7 +383,7 @@ io.lettuce lettuce-core - 6.0.2.RELEASE + 6.5.1.RELEASE org.apache.commons diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 0382cfafee6..f1e28382da6 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -4,7 +4,7 @@ "private": true, "dependencies": { "@elastic/datemath": "^5.0.3", - "@elastic/eui": "^55.0.1", + "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", "@feast-dev/feast-ui": "0.42.0", "@testing-library/jest-dom": "^5.16.4", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 2e6af4dc7ca..065f75d8ea5 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1272,10 +1272,10 @@ dependencies: tslib "^1.9.3" -"@elastic/eui@^55.0.1": - version "55.1.2" - resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-55.1.2.tgz#dd0b42f5b26c5800d6a9cb2d4c2fe1afce9d3f07" - integrity sha512-wwZz5KxMIMFlqEsoCRiQBJDc4CrluS1d0sCOmQ5lhIzKhYc91MdxnqCk2i6YkhL4sSDf2Y9KAEuMXa+uweOWUA== +"@elastic/eui@^72.0.0": + version "72.2.0" + resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-72.2.0.tgz#0d89ec4c6d8a677ba41d086abd509c5a5ea09180" + integrity sha512-3JHKLWqbU1A6qMVkw0n1VZ5PaL07sd3N44tWsRCn+DEaDv9jq68ilEmY1wdYqKXw8VyFwcPbd8ZYZpdzBD2nPA== dependencies: "@types/chroma-js" "^2.0.0" "@types/lodash" "^4.14.160" @@ -1296,7 +1296,7 @@ react-beautiful-dnd "^13.1.0" react-dropzone "^11.5.3" react-element-to-jsx-string "^14.3.4" - react-focus-on "^3.5.4" + react-focus-on "^3.7.0" react-input-autosize "^3.0.0" react-is "^17.0.2" react-virtualized-auto-sizer "^1.0.6" @@ -1307,7 +1307,7 @@ rehype-stringify "^8.0.0" remark-breaks "^2.0.2" remark-emoji "^2.1.0" - remark-parse "^8.0.3" + remark-parse-no-trim "^8.0.4" remark-rehype "^8.0.0" tabbable "^5.2.1" text-diff "^1.0.1" @@ -3363,13 +3363,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -aria-hidden@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.1.3.tgz#bb48de18dc84787a3c6eee113709c473c64ec254" - integrity sha512-RhVWFtKH5BiGMycI72q2RAFMLQi8JP9bLuQXgR5a8Znp7P5KOIADSJeyfI8PCVxLEp067B2HbP5JIiI/PXIZeA== - dependencies: - tslib "^1.0.0" - aria-hidden@^1.2.2: version "1.2.4" resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.2.4.tgz#b78e383fdbc04d05762c78b4a25a501e736c4522" @@ -5724,13 +5717,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== -focus-lock@^0.11.2: - version "0.11.2" - resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-0.11.2.tgz#aeef3caf1cea757797ac8afdebaec8fd9ab243ed" - integrity sha512-pZ2bO++NWLHhiKkgP1bEXHhR1/OjVcSvlCJ98aNJDFeb7H5OOQaO+SKOZle6041O9rv2tmbrO4JzClAvDUHf0g== - dependencies: - tslib "^2.0.3" - focus-lock@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-1.3.5.tgz#aa644576e5ec47d227b57eb14e1efb2abf33914c" @@ -9103,32 +9089,7 @@ react-focus-lock@^2.11.3: use-callback-ref "^1.3.2" use-sidecar "^1.1.2" -react-focus-lock@^2.9.0: - version "2.9.1" - resolved "https://registry.yarnpkg.com/react-focus-lock/-/react-focus-lock-2.9.1.tgz#094cfc19b4f334122c73bb0bff65d77a0c92dd16" - integrity sha512-pSWOQrUmiKLkffPO6BpMXN7SNKXMsuOakl652IBuALAu1esk+IcpJyM+ALcYzPTTFz1rD0R54aB9A4HuP5t1Wg== - dependencies: - "@babel/runtime" "^7.0.0" - focus-lock "^0.11.2" - prop-types "^15.6.2" - react-clientside-effect "^1.2.6" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.5.4: - version "3.6.0" - resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.6.0.tgz#159e13082dad4ea1f07abe11254f0e981d5a7b79" - integrity sha512-onIRjpd9trAUenXNdDcvjc8KJUSklty4X/Gr7hAm/MzM7ekSF2pg9D8KBKL7ipige22IAPxLRRf/EmJji9KD6Q== - dependencies: - aria-hidden "^1.1.3" - react-focus-lock "^2.9.0" - react-remove-scroll "^2.5.2" - react-style-singleton "^2.2.0" - tslib "^2.3.1" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.9.1: +react-focus-on@^3.7.0, react-focus-on@^3.9.1: version "3.9.4" resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.9.4.tgz#0b6c13273d86243c330d1aa53af39290f543da7b" integrity sha512-NFKmeH6++wu8e7LJcbwV8TTd4L5w/U5LMXTMOdUcXhCcZ7F5VOvgeTHd4XN1PD7TNmdvldDu/ENROOykUQ4yQg== @@ -9209,14 +9170,6 @@ react-refresh@^0.11.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046" integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A== -react-remove-scroll-bar@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.1.tgz#9f13b05b249eaa57c8d646c1ebb83006b3581f5f" - integrity sha512-IvGX3mJclEF7+hga8APZczve1UyGMkMG+tjS0o/U1iLgvZRpjFAQEUBJ4JETfvbNlfNnZnoDyWJCICkA15Mghg== - dependencies: - react-style-singleton "^2.2.0" - tslib "^2.0.0" - react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz#3e585e9d163be84a010180b18721e851ac81a29c" @@ -9225,17 +9178,6 @@ react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: react-style-singleton "^2.2.1" tslib "^2.0.0" -react-remove-scroll@^2.5.2: - version "2.5.3" - resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.5.3.tgz#a152196e710e8e5811be39dc352fd8a90b05c961" - integrity sha512-NQ1bXrxKrnK5pFo/GhLkXeo3CrK5steI+5L+jynwwIemvZyfXqaL0L5BzwJd7CSwNCU723DZaccvjuyOdoy3Xw== - dependencies: - react-remove-scroll-bar "^2.3.1" - react-style-singleton "^2.2.0" - tslib "^2.0.0" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - react-remove-scroll@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz#fb03a0845d7768a4f1519a99fdb84983b793dc07" @@ -9317,15 +9259,6 @@ react-scripts@^5.0.0: optionalDependencies: fsevents "^2.3.2" -react-style-singleton@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.0.tgz#70f45f5fef97fdb9a52eed98d1839fa6b9032b22" - integrity sha512-nK7mN92DMYZEu3cQcAhfwE48NpzO5RpxjG4okbSqRRbfal9Pk+fG2RdQXTMp+f6all1hB9LIJSt+j7dCYrU11g== - dependencies: - get-nonce "^1.0.0" - invariant "^2.2.4" - tslib "^2.0.0" - react-style-singleton@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4" @@ -9589,28 +9522,6 @@ remark-parse-no-trim@^8.0.4: vfile-location "^3.0.0" xtend "^4.0.1" -remark-parse@^8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-8.0.3.tgz#9c62aa3b35b79a486454c690472906075f40c7e1" - integrity sha512-E1K9+QLGgggHxCQtLt++uXltxEprmWzNfg+MxpfHsZlrddKzZ/hZyWHDbK3/Ap8HJQqYJRXP+jHczdL6q6i85Q== - dependencies: - ccount "^1.0.0" - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^2.0.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^2.0.0" - vfile-location "^3.0.0" - xtend "^4.0.1" - remark-rehype@^8.0.0, remark-rehype@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-8.1.0.tgz#610509a043484c1e697437fa5eb3fd992617c945" @@ -10643,11 +10554,6 @@ trim-trailing-lines@^1.0.0: resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0" integrity sha512-rjUWSqnfTNrjbB9NQWfPMH/xRK1deHeGsHoVfpxJ++XeYXE0d6B1En37AHfw3jtfTU7dzMzZL2jjpe8Qb5gLIQ== -trim@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= - trough@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" @@ -10673,7 +10579,7 @@ tslib@2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.3: +tslib@^1.8.1, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index fff0993b1d0..c0b1c348a3a 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -69,11 +71,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -88,7 +90,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -128,9 +130,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -146,21 +148,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -176,7 +178,7 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -193,9 +195,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -213,7 +215,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -222,7 +224,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -244,9 +246,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -266,7 +268,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -347,7 +349,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -375,7 +377,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -407,7 +409,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -420,7 +422,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -445,7 +447,7 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 +marshmallow==3.23.2 # via # environs # great-expectations @@ -457,7 +459,7 @@ mdurl==0.1.2 # via markdown-it-py milvus-lite==2.4.10 # via pymilvus -minio==7.1.0 +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -487,7 +489,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -501,7 +503,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -590,13 +592,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -667,13 +669,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.2 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -682,7 +686,7 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal @@ -696,7 +700,7 @@ pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -752,7 +756,7 @@ python-dotenv==1.0.1 # via # environs # uvicorn -python-json-logger==2.0.7 +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -828,7 +832,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -838,7 +842,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -856,9 +860,8 @@ setuptools==75.6.0 # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -873,7 +876,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -923,7 +926,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -969,7 +972,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -983,7 +986,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -999,7 +1002,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1015,6 +1018,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg @@ -1050,7 +1054,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1062,7 +1066,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1092,7 +1096,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 63a887e1aac..87b9cf04c91 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,19 +25,19 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -85,25 +85,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.2 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -122,11 +122,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -140,7 +140,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -164,7 +164,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -172,7 +172,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 4dbfc44509b..37bcbdb2c9c 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -44,7 +46,7 @@ async-timeout==5.0.1 # via redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -67,11 +69,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -86,7 +88,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -126,9 +128,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -144,21 +146,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -174,7 +176,7 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -186,9 +188,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -206,7 +208,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -215,7 +217,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -237,9 +239,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -259,7 +261,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -338,7 +340,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -366,7 +368,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -398,7 +400,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -411,7 +413,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -436,7 +438,7 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 +marshmallow==3.23.2 # via # environs # great-expectations @@ -448,7 +450,7 @@ mdurl==0.1.2 # via markdown-it-py milvus-lite==2.4.10 # via pymilvus -minio==7.1.0 +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -478,7 +480,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -492,7 +494,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -581,13 +583,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -658,13 +660,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.2 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -673,7 +677,7 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal @@ -687,7 +691,7 @@ pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -743,7 +747,7 @@ python-dotenv==1.0.1 # via # environs # uvicorn -python-json-logger==2.0.7 +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -819,7 +823,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -829,7 +833,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -847,9 +851,8 @@ setuptools==75.6.0 # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -864,7 +867,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -950,7 +953,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -964,7 +967,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -980,12 +983,13 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via + # anyio # azure-core # azure-identity # azure-storage-blob @@ -994,6 +998,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # mypy # psycopg # psycopg-pool @@ -1026,7 +1031,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1038,7 +1043,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1068,7 +1073,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 42f89ecb6a5..c536ef91ae3 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,17 +25,17 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -83,25 +83,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.2 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -120,11 +120,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -148,6 +148,7 @@ typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via + # anyio # fastapi # mypy # pydantic @@ -158,7 +159,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -166,7 +167,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 7357f3741f1..da914388de2 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==4.1.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -71,11 +73,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -90,7 +92,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -130,9 +132,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -156,13 +158,13 @@ dask-expr==1.1.10 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -178,7 +180,7 @@ duckdb==0.10.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -195,9 +197,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -215,7 +217,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -224,7 +226,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -246,9 +248,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -268,7 +270,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -384,7 +386,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -416,7 +418,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -429,7 +431,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -454,7 +456,7 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 +marshmallow==3.23.2 # via # environs # great-expectations @@ -466,7 +468,7 @@ mdurl==0.1.2 # via markdown-it-py milvus-lite==2.4.10 # via pymilvus -minio==7.1.0 +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -496,7 +498,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -510,7 +512,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -598,13 +600,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -675,13 +677,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.2 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -690,7 +694,7 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal @@ -704,7 +708,7 @@ pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -760,7 +764,7 @@ python-dotenv==1.0.1 # via # environs # uvicorn -python-json-logger==2.0.7 +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -836,7 +840,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -846,7 +850,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -864,9 +868,8 @@ setuptools==75.6.0 # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -881,7 +884,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -931,7 +934,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -977,7 +980,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -991,7 +994,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -1007,7 +1010,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1024,12 +1027,14 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg # psycopg-pool # pydantic # pydantic-core + # python-json-logger # rich # snowflake-connector-python # sqlalchemy @@ -1061,7 +1066,7 @@ urllib3==1.26.20 # responses # snowflake-connector-python # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1073,7 +1078,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1103,7 +1108,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index db7113dc2c2..80f1e499e1f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -35,9 +35,9 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -87,25 +87,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.2 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -124,11 +124,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -142,7 +142,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -167,7 +167,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -175,7 +175,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/setup.py b/setup.py index 7593be4c376..59b881c9715 100644 --- a/setup.py +++ b/setup.py @@ -162,14 +162,14 @@ [ "build", "virtualenv==20.23.0", - "cryptography>=35.0,<43", + "cryptography>=43.0,<44", "ruff>=0.8.0", "mypy-protobuf>=3.1", "grpcio-tools>=1.56.2,<2", "grpcio-testing>=1.56.2,<2", # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). - "httpx>=0.23.3", - "minio==7.1.0", + "httpx==0.27.2", + "minio==7.2.11", "mock==2.0.0", "moto<5", "mypy>=1.4.1,<1.11.3", From 7df287e8c0f5ec3ab3fa88fd5576f636053a3769 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Sat, 21 Dec 2024 10:43:50 -0500 Subject: [PATCH 53/90] =?UTF-8?q?feat:=20Adding=20features=20field=20to=20?= =?UTF-8?q?retrieve=5Fonline=5Ffeatures=20to=20return=20mor=E2=80=A6=20(#4?= =?UTF-8?q?869)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdk/python/feast/feature_store.py | 56 ++++++++++++++----- sdk/python/feast/infra/key_encoding_utils.py | 17 +++++- .../elasticsearch.py | 3 +- .../infra/online_stores/faiss_online_store.py | 3 +- .../feast/infra/online_stores/online_store.py | 8 ++- .../postgres_online_store/postgres.py | 4 +- .../qdrant_online_store/qdrant.py | 3 +- .../feast/infra/online_stores/sqlite.py | 26 +++------ .../feast/infra/passthrough_provider.py | 4 +- sdk/python/feast/infra/provider.py | 4 +- sdk/python/feast/utils.py | 4 ++ 11 files changed, 91 insertions(+), 41 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index edbd060e106..4564d6abf3c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1753,9 +1753,10 @@ async def get_online_features_async( def retrieve_online_documents( self, - feature: str, + feature: Optional[str], query: Union[str, List[float]], top_k: int, + features: Optional[List[str]] = None, distance_metric: Optional[str] = None, ) -> OnlineResponse: """ @@ -1765,6 +1766,7 @@ def retrieve_online_documents( feature: The list of document features that should be retrieved from the online document store. These features can be specified either as a list of string document feature references or as a feature service. String feature references must have format "feature_view:feature", e.g, "document_fv:document_embeddings". + features: The list of features that should be retrieved from the online store. query: The query to retrieve the closest document features for. top_k: The number of closest document features to retrieve. distance_metric: The distance metric to use for retrieval. @@ -1773,18 +1775,44 @@ def retrieve_online_documents( raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) + feature_list: List[str] = ( + features + if features is not None + else ([feature] if feature is not None else []) + ) + ( available_feature_views, _, ) = utils._get_feature_views_to_use( registry=self._registry, project=self.project, - features=[feature], + features=feature_list, allow_cache=True, hide_dummy_entity=False, ) + if features: + feature_view_set = set() + for feature in features: + feature_view_name = feature.split(":")[0] + feature_view = self.get_feature_view(feature_view_name) + feature_view_set.add(feature_view.name) + if len(feature_view_set) > 1: + raise ValueError( + "Document retrieval only supports a single feature view." + ) + requested_feature = None + requested_features = [ + f.split(":")[1] for f in features if isinstance(f, str) and ":" in f + ] + else: + requested_feature = ( + feature.split(":")[1] if isinstance(feature, str) else feature + ) + requested_features = [requested_feature] if requested_feature else [] + requested_feature_view_name = ( - feature.split(":")[0] if isinstance(feature, str) else feature + feature.split(":")[0] if feature else list(feature_view_set)[0] ) for feature_view in available_feature_views: if feature_view.name == requested_feature_view_name: @@ -1793,14 +1821,15 @@ def retrieve_online_documents( raise ValueError( f"Feature view {requested_feature_view} not found in the registry." ) - requested_feature = ( - feature.split(":")[1] if isinstance(feature, str) else feature - ) + + requested_feature_view = available_feature_views[0] + provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, requested_feature_view, requested_feature, + requested_features, query, top_k, distance_metric, @@ -1822,6 +1851,7 @@ def retrieve_online_documents( document_feature_vals = [feature[4] for feature in document_features] document_feature_distance_vals = [feature[5] for feature in document_features] online_features_response = GetOnlineFeaturesResponse(results=[]) + requested_feature = requested_feature or requested_features[0] utils._populate_result_rows_from_columnar( online_features_response=online_features_response, data={ @@ -1836,7 +1866,8 @@ def _retrieve_from_online_store( self, provider: Provider, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str], @@ -1852,6 +1883,7 @@ def _retrieve_from_online_store( config=self.config, table=table, requested_feature=requested_feature, + requested_features=requested_features, query=query, top_k=top_k, distance_metric=distance_metric, @@ -1952,19 +1984,13 @@ def serve_ui( ) def serve_registry( - self, - port: int, - tls_key_path: str = "", - tls_cert_path: str = "", + self, port: int, tls_key_path: str = "", tls_cert_path: str = "" ) -> None: """Start registry server locally on a given port.""" from feast import registry_server registry_server.start_server( - self, - port=port, - tls_key_path=tls_key_path, - tls_cert_path=tls_cert_path, + self, port=port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path ) def serve_offline( diff --git a/sdk/python/feast/infra/key_encoding_utils.py b/sdk/python/feast/infra/key_encoding_utils.py index 1f9ffeef140..18127896bd5 100644 --- a/sdk/python/feast/infra/key_encoding_utils.py +++ b/sdk/python/feast/infra/key_encoding_utils.py @@ -1,5 +1,7 @@ import struct -from typing import List, Tuple +from typing import List, Tuple, Union + +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -163,3 +165,16 @@ def get_list_val_str(val): if val.HasField(accept_type): return str(getattr(val, accept_type).val) return None + + +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack(f"{vector_length}f", *vector) + + +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) diff --git a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py index 0152ca330c9..af328141520 100644 --- a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py +++ b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py @@ -213,7 +213,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, *args, diff --git a/sdk/python/feast/infra/online_stores/faiss_online_store.py b/sdk/python/feast/infra/online_stores/faiss_online_store.py index cc2e75800e6..fd4d6768abd 100644 --- a/sdk/python/feast/infra/online_stores/faiss_online_store.py +++ b/sdk/python/feast/infra/online_stores/faiss_online_store.py @@ -176,7 +176,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featres: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 789885f82bc..be3128562dc 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -390,7 +390,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -411,6 +412,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose feature values should be read. requested_feature: The name of the feature whose embeddings should be used for retrieval. + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The embeddings to use for retrieval. top_k: The number of documents to retrieve. @@ -419,6 +421,10 @@ def retrieve_online_documents( where the first item is the event timestamp for the row, and the second item is a dict of feature name to embeddings. """ + if not requested_feature and not requested_features: + raise ValueError( + "Either requested_feature or requested_features must be specified" + ) raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) diff --git a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py index 7c099c80ecc..f43247a5457 100644 --- a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py +++ b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py @@ -347,7 +347,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "L2", @@ -366,6 +367,7 @@ def retrieve_online_documents( config: Feast configuration object table: FeatureView object as the table to search requested_feature: The requested feature as the column to search + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The query embedding to search for top_k: The number of items to return distance_metric: The distance metric to use for the search.G diff --git a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py index 074c52ba5e8..cdbef95348d 100644 --- a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py +++ b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py @@ -248,7 +248,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "cosine", diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index e2eeb038d00..23b4f6db3a3 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -15,19 +15,20 @@ import logging import os import sqlite3 -import struct import sys from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple -from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity from feast.feature_view import FeatureView from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.key_encoding_utils import ( + serialize_entity_key, + serialize_f32, +) from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto @@ -330,7 +331,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featuers: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -432,6 +434,7 @@ def retrieve_online_documents( _build_retrieve_online_document_record( entity_key, string_value if string_value else b"", + # This may be a bug embedding, distance, event_ts, @@ -459,19 +462,6 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32( - vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int -) -> bytes: - """serializes a list of floats into a compact "raw bytes" format""" - return struct.pack(f"{vector_length}f", *vector) - - -def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: - """deserializes a list of floats from a compact "raw bytes" format""" - num_floats = vector_length // 4 # 4 bytes per float - return list(struct.unpack(f"{num_floats}f", byte_vector)) - - class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 215b175eb2e..57aa122ae8a 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -294,7 +294,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -305,6 +306,7 @@ def retrieve_online_documents( config, table, requested_feature, + requested_features, query, top_k, distance_metric, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 8351f389ad9..efc806ba2f0 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -419,7 +419,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -440,6 +441,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_feature: the requested document feature name. + requested_features: the requested document feature names. query: The query embedding to search for. top_k: The number of documents to return. diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 51d4bf4f2cc..cfc19e37ca4 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1192,6 +1192,10 @@ def _utc_now() -> datetime: return datetime.now(tz=timezone.utc) +def _serialize_vector_to_float_list(vector: List[float]) -> ValueProto: + return ValueProto(float_list_val=FloatListProto(val=vector)) + + def _build_retrieve_online_document_record( entity_key: Union[str, bytes], feature_value: Union[str, bytes], From 3dbd58b4bfa94a633e2047fd3864fb10ff6c2353 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Sat, 21 Dec 2024 21:18:53 -0500 Subject: [PATCH 54/90] chore: Update document test to add other feature data types and fix feature logging (#4872) Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/data/data_creator.py | 2 ++ .../feature_repos/universal/feature_views.py | 18 +++++++++++++++--- .../offline_store/test_feature_logging.py | 8 ++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 6b0984f799d..dfe94913e97 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -84,6 +84,8 @@ def get_feature_values_for_dtype( def create_document_dataset() -> pd.DataFrame: data = { "item_id": [1, 2, 3], + "string_feature": ["a", "b", "c"], + "float_feature": [1.0, 2.0, 3.0], "embedding_float": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "embedding_double": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "ts": [ diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 11ddcb0ecc6..47e5746e61a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,7 +17,7 @@ from feast.data_source import DataSource, RequestSource from feast.feature_view_projection import FeatureViewProjection from feast.on_demand_feature_view import PandasTransformation, SubstraitTransformation -from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 +from feast.types import Array, FeastType, Float32, Float64, Int32, Int64, String from tests.integration.feature_repos.universal.entities import ( customer, driver, @@ -160,8 +160,20 @@ def create_item_embeddings_feature_view(source, infer_features: bool = False): schema=None if infer_features else [ - Field(name="embedding_double", dtype=Array(Float64)), - Field(name="embedding_float", dtype=Array(Float32)), + Field( + name="embedding_double", + dtype=Array(Float64), + vector_index=True, + vector_search_metric="L2", + ), + Field( + name="embedding_float", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="string_feature", dtype=String), + Field(name="float_feature", dtype=Float32), ], source=source, ttl=timedelta(hours=2), diff --git a/sdk/python/tests/integration/offline_store/test_feature_logging.py b/sdk/python/tests/integration/offline_store/test_feature_logging.py index 32f506f90b2..53147d242ef 100644 --- a/sdk/python/tests/integration/offline_store/test_feature_logging.py +++ b/sdk/python/tests/integration/offline_store/test_feature_logging.py @@ -106,7 +106,15 @@ def retrieve(): ) persisted_logs = persisted_logs[expected_columns] + logs_df = logs_df[expected_columns] + + # Convert timezone-aware datetime values to naive datetime values + logs_df[LOG_TIMESTAMP_FIELD] = logs_df[LOG_TIMESTAMP_FIELD].dt.tz_localize(None) + persisted_logs[LOG_TIMESTAMP_FIELD] = persisted_logs[ + LOG_TIMESTAMP_FIELD + ].dt.tz_localize(None) + pd.testing.assert_frame_equal( logs_df.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), persisted_logs.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), From 8f591a235ba5bd9d1bc598195f46c7e12e437a2c Mon Sep 17 00:00:00 2001 From: nanohanno <44575187+nanohanno@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:23:25 +0100 Subject: [PATCH 55/90] feat: Use ASOF JOIN in Snowflake offline store query (#4850) * Use ASOF JOIN in Snowflake offline store query Signed-off-by: hkuepers * Fix Snowflake query template for entityless feature views Signed-off-by: hkuepers * Remove quotes on subquery in snowflake template Signed-off-by: hkuepers * Use __subquery in Snowflake template for preparation Signed-off-by: hkuepers * Fix deduplication in Snowflake query string Signed-off-by: hkuepers * Use event_timestamp in ttl cte Signed-off-by: hkuepers --------- Signed-off-by: hkuepers Co-authored-by: hkuepers --- .../feast/infra/offline_stores/snowflake.py | 130 ++++++------------ 1 file changed, 44 insertions(+), 86 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 3d23682769b..101685cec6f 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -716,8 +716,8 @@ def _get_entity_df_event_timestamp_range( MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN = """ /* - Compute a deterministic hash for the `left_table_query_string` that will be used throughout - all the logic as the field to GROUP BY the data + 0. Compute a deterministic hash for the `left_table_query_string` that will be used throughout + all the logic as the field to GROUP BY the data. */ WITH "entity_dataframe" AS ( SELECT *, @@ -739,6 +739,10 @@ def _get_entity_df_event_timestamp_range( {% for featureview in featureviews %} +/* + 1. Only select the required columns with entities of the featureview. +*/ + "{{ featureview.name }}__entity_dataframe" AS ( SELECT {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} @@ -752,20 +756,7 @@ def _get_entity_df_event_timestamp_range( ), /* - This query template performs the point-in-time correctness join for a single feature set table - to the provided entity table. - - 1. We first join the current feature_view to the entity dataframe that has been passed. - This JOIN has the following logic: - - For each row of the entity dataframe, only keep the rows where the `timestamp_field` - is less than the one provided in the entity dataframe - - If there a TTL for the current feature_view, also keep the rows where the `timestamp_field` - is higher the the one provided minus the TTL - - For each row, Join on the entity key and retrieve the `entity_row_unique_id` that has been - computed previously - - The output of this CTE will contain all the necessary information and already filtered out most - of the data that is not relevant. +2. Use subquery to prepare event_timestamp, created_timestamp, entity columns and feature columns. */ "{{ featureview.name }}__subquery" AS ( @@ -777,94 +768,61 @@ def _get_entity_df_event_timestamp_range( "{{ feature }}" as {% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} - WHERE "{{ featureview.timestamp_field }}" <= '{{ featureview.max_event_timestamp }}' - {% if featureview.ttl == 0 %}{% else %} - AND "{{ featureview.timestamp_field }}" >= '{{ featureview.min_event_timestamp }}' - {% endif %} -), - -"{{ featureview.name }}__base" AS ( - SELECT - "subquery".*, - "entity_dataframe"."entity_timestamp", - "entity_dataframe"."{{featureview.name}}__entity_row_unique_id" - FROM "{{ featureview.name }}__subquery" AS "subquery" - INNER JOIN "{{ featureview.name }}__entity_dataframe" AS "entity_dataframe" - ON TRUE - AND "subquery"."event_timestamp" <= "entity_dataframe"."entity_timestamp" - - {% if featureview.ttl == 0 %}{% else %} - AND "subquery"."event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_dataframe"."entity_timestamp") - {% endif %} - - {% for entity in featureview.entities %} - AND "subquery"."{{ entity }}" = "entity_dataframe"."{{ entity }}" - {% endfor %} ), /* - 2. If the `created_timestamp_column` has been set, we need to - deduplicate the data first. This is done by calculating the - `MAX(created_at_timestamp)` for each event_timestamp. - We then join the data on the next CTE +3. If the `created_timestamp_column` has been set, we need to +deduplicate the data first. This is done by calculating the +`MAX(created_at_timestamp)` for each event_timestamp and joining back on the subquery. +Otherwise, the ASOF JOIN can have unstable side effects +https://docs.snowflake.com/en/sql-reference/constructs/asof-join#expected-behavior-when-ties-exist-in-the-right-table */ + {% if featureview.created_timestamp_column %} "{{ featureview.name }}__dedup" AS ( - SELECT - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp", - MAX("created_timestamp") AS "created_timestamp" - FROM "{{ featureview.name }}__base" - GROUP BY "{{featureview.name}}__entity_row_unique_id", "event_timestamp" + SELECT * + FROM "{{ featureview.name }}__subquery" + INNER JOIN ( + SELECT + {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} + "event_timestamp", + MAX("created_timestamp") AS "created_timestamp" + FROM "{{ featureview.name }}__subquery" + GROUP BY {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp" + ) + USING({{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp", "created_timestamp") ), {% endif %} /* - 3. The data has been filtered during the first CTE "*__base" - Thus we only need to compute the latest timestamp of each feature. +4. Make ASOF JOIN of deduplicated feature CTE on reduced entity dataframe. */ -"{{ featureview.name }}__latest" AS ( + +"{{ featureview.name }}__asof_join" AS ( SELECT - "event_timestamp", - {% if featureview.created_timestamp_column %}"created_timestamp",{% endif %} - "{{featureview.name}}__entity_row_unique_id" - FROM - ( - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY "{{featureview.name}}__entity_row_unique_id" - ORDER BY "event_timestamp" DESC{% if featureview.created_timestamp_column %},"created_timestamp" DESC{% endif %} - ) AS "row_number" - FROM "{{ featureview.name }}__base" - {% if featureview.created_timestamp_column %} - INNER JOIN "{{ featureview.name }}__dedup" - USING ("{{featureview.name}}__entity_row_unique_id", "event_timestamp", "created_timestamp") - {% endif %} - ) - WHERE "row_number" = 1 + e.*, + v.* + FROM "{{ featureview.name }}__entity_dataframe" e + ASOF JOIN {% if featureview.created_timestamp_column %}"{{ featureview.name }}__dedup"{% else %}"{{ featureview.name }}__subquery"{% endif %} v + MATCH_CONDITION (e."entity_timestamp" >= v."event_timestamp") + {% if featureview.entities %} USING({{ featureview.entities | map('tojson') | join(', ')}}) {% endif %} ), /* - 4. Once we know the latest value of each feature for a given timestamp, - we can join again the data back to the original "base" dataset +5. If TTL is configured filter the CTE to remove rows where the feature values are older than the configured ttl. */ -"{{ featureview.name }}__cleaned" AS ( - SELECT "base".* - FROM "{{ featureview.name }}__base" AS "base" - INNER JOIN "{{ featureview.name }}__latest" - USING( - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp" - {% if featureview.created_timestamp_column %} - ,"created_timestamp" - {% endif %} - ) -){% if loop.last %}{% else %}, {% endif %} +"{{ featureview.name }}__ttl" AS ( + SELECT * + FROM "{{ featureview.name }}__asof_join" + {% if featureview.ttl == 0 %}{% else %} + WHERE "event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_timestamp") + {% endif %} +){% if loop.last %}{% else %}, {% endif %} {% endfor %} /* - Joins the outputs of multiple time travel joins to a single table. + Join the outputs of multiple time travel joins to a single table. The entity_dataframe dataset being our source of truth here. */ @@ -877,7 +835,7 @@ def _get_entity_df_event_timestamp_range( {% for feature in featureview.features %} ,{% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %} {% endfor %} - FROM "{{ featureview.name }}__cleaned" -) "{{ featureview.name }}__cleaned" USING ("{{featureview.name}}__entity_row_unique_id") + FROM "{{ featureview.name }}__ttl" +) "{{ featureview.name }}__ttl" USING ("{{featureview.name}}__entity_row_unique_id") {% endfor %} """ From 4de187de33fe361cea60bd19011e26b7f8ea5390 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:33:17 +0000 Subject: [PATCH 56/90] chore: Bump nanoid from 3.3.4 to 3.3.8 in /sdk/python/feast/ui (#4831) Bumps [nanoid](https://github.com/ai/nanoid) from 3.3.4 to 3.3.8. - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/3.3.4...3.3.8) --- updated-dependencies: - dependency-name: nanoid dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/yarn.lock | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 065f75d8ea5..6578b4a34f7 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -7771,15 +7771,10 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" -nanoid@^3.3.3: - version "3.3.4" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== - -nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +nanoid@^3.3.3, nanoid@^3.3.7: + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== natural-compare@^1.4.0: version "1.4.0" From 3f49517dfeabea5ffbd3f6b589cc0f2280ee4018 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Mon, 23 Dec 2024 11:35:02 -0600 Subject: [PATCH 57/90] fix: Improve status.applied updates & add offline pvc unit test (#4871) improve status.applied update & add offline pvc unit test Signed-off-by: Tommy Hughes --- .../controller/services/repo_config.go | 4 +- .../controller/services/repo_config_test.go | 30 ++++++++- .../internal/controller/services/util.go | 64 +++++++++---------- 3 files changed, 60 insertions(+), 38 deletions(-) diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 675fbd047f8..0a5dd11544e 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -85,17 +85,17 @@ func getBaseServiceRepoConfig( featureStore *feastdevv1alpha1.FeatureStore, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { - appliedSpec := featureStore.Status.Applied repoConfig := defaultRepoConfig(featureStore) clientRepoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) if err != nil { return repoConfig, err } - repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig if isRemoteRegistry(featureStore) { repoConfig.Registry = clientRepoConfig.Registry } + repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig + appliedSpec := featureStore.Status.Applied if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") if authSecretErr != nil { diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 42525700a4c..9138f00f2f6 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -46,7 +46,6 @@ var _ = Describe("Repo Config", func() { Path: EphemeralPath + "/" + DefaultOnlineStorePath, } - var repoConfig RepoConfig repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) @@ -100,6 +99,35 @@ var _ = Describe("Repo Config", func() { Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) + By("Having an offlineStore with PVC") + mountPath := "/testing" + expectedOnlineConfig.Path = mountPath + "/" + DefaultOnlineStorePath + expectedRegistryConfig.Path = mountPath + "/" + DefaultRegistryPath + + featureStore = minimalFeatureStore() + featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + OfflineStore: &feastdevv1alpha1.OfflineStore{ + Persistence: &feastdevv1alpha1.OfflineStorePersistence{ + FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ + PvcConfig: &feastdevv1alpha1.PvcConfig{ + MountPath: mountPath, + }, + }, + }, + }, + } + ApplyDefaultsToStatus(featureStore) + appliedServices := featureStore.Status.Applied.Services + Expect(appliedServices.OnlineStore).To(BeNil()) + Expect(appliedServices.Registry.Local.Persistence.FilePersistence.Path).To(Equal(expectedRegistryConfig.Path)) + + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.OfflineStore).To(Equal(defaultOfflineStoreConfig)) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + By("Having the all the file services") featureStore = minimalFeatureStore() featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index d0ca94ff865..7b9c177c89d 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -37,15 +37,18 @@ func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastSe if services != nil { switch feastType { case OnlineFeastType: - if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { + if services.OnlineStore != nil && services.OnlineStore.Persistence != nil && + services.OnlineStore.Persistence.FilePersistence != nil { pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig } case OfflineFeastType: - if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { + if services.OfflineStore != nil && services.OfflineStore.Persistence != nil && + services.OfflineStore.Persistence.FilePersistence != nil { pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig } case RegistryFeastType: - if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { + if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence != nil && + services.Registry.Local.Persistence.FilePersistence != nil { pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig } } @@ -66,18 +69,18 @@ func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { } func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { - if featureStore.Status.Applied.Services != nil { - if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { - return pvcConfig.MountPath - } + if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { + return pvcConfig.MountPath } return EphemeralPath } func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { + // overwrite status.applied with every reconcile + cr.Spec.DeepCopyInto(&cr.Status.Applied) cr.Status.FeastVersion = feastversion.FeastVersion - applied := cr.Spec.DeepCopy() + applied := &cr.Status.Applied if applied.Services == nil { applied.Services = &feastdevv1alpha1.FeatureStoreServices{} } @@ -106,10 +109,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(cr) } - if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.Registry.Local.Persistence.FilePersistence.PvcConfig - ensurePVCDefaults(pvc, RegistryFeastType) - } + ensurePVCDefaults(services.Registry.Local.Persistence.FilePersistence.PvcConfig, RegistryFeastType) } setServiceDefaultConfigs(&services.Registry.Local.ServiceConfigs.DefaultConfigs) @@ -131,10 +131,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { services.OfflineStore.Persistence.FilePersistence.Type = string(OfflineFilePersistenceDaskConfigType) } - if services.OfflineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OfflineStore.Persistence.FilePersistence.PvcConfig - ensurePVCDefaults(pvc, OfflineFeastType) - } + ensurePVCDefaults(services.OfflineStore.Persistence.FilePersistence.PvcConfig, OfflineFeastType) } setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) @@ -154,16 +151,11 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) } - if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OnlineStore.Persistence.FilePersistence.PvcConfig - ensurePVCDefaults(pvc, OnlineFeastType) - } + ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) } setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) } - // overwrite status.applied with every reconcile - applied.DeepCopyInto(&cr.Status.Applied) } func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { @@ -189,19 +181,21 @@ func ensureRequestedStorage(resources *v1.VolumeResourceRequirements, requestedS } func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { - var storageRequest string - switch feastType { - case OnlineFeastType: - storageRequest = DefaultOnlineStorageRequest - case OfflineFeastType: - storageRequest = DefaultOfflineStorageRequest - case RegistryFeastType: - storageRequest = DefaultRegistryStorageRequest - } - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, storageRequest) - if pvc.Create.AccessModes == nil { - pvc.Create.AccessModes = DefaultPVCAccessModes + if pvc != nil { + var storageRequest string + switch feastType { + case OnlineFeastType: + storageRequest = DefaultOnlineStorageRequest + case OfflineFeastType: + storageRequest = DefaultOfflineStorageRequest + case RegistryFeastType: + storageRequest = DefaultRegistryStorageRequest + } + if pvc.Create != nil { + ensureRequestedStorage(&pvc.Create.Resources, storageRequest) + if pvc.Create.AccessModes == nil { + pvc.Create.AccessModes = DefaultPVCAccessModes + } } } } From 18ff604b07d0874faac275886e559c982a7aafe4 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 23 Dec 2024 14:20:33 -0500 Subject: [PATCH 58/90] chore: Update unit_tests.yml to use latest Mac OS (#4875) * chore: Update unit_tests.yml to use latest Mac OS * Update unit_tests.yml * Update unit_tests.yml * Update unit_tests.yml --- .github/workflows/unit_tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 443f40270ff..3ece863de3b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -13,10 +13,15 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10", "3.11"] - os: [ ubuntu-latest, macos-13 ] + os: [ ubuntu-latest, macos-13, macos-14 ] exclude: - os: macos-13 python-version: "3.9" + - os: macos-14 + python-version: "3.9" + - os: macos-14 + python-version: "3.10" + env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} From 22c7b58f9590a357eaa57c77d5ed351f1fa07501 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 23 Dec 2024 20:56:05 -0500 Subject: [PATCH 59/90] feat: Add Milvus Vector Database Implementation (#4751) --- .../workflows/pr_local_integration_tests.yml | 2 +- Makefile | 18 +- .../adding-support-for-a-new-online-store.md | 2 +- ...nfra.online_stores.milvus_online_store.rst | 8 + .../milvus_online_store/milvus.py | 428 ++++++++++++++++++ sdk/python/feast/repo_config.py | 1 + sdk/python/feast/type_map.py | 40 +- sdk/python/tests/foo_provider.py | 1 + .../online_store/test_universal_online.py | 29 ++ 9 files changed, 505 insertions(+), 24 deletions(-) create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 2825b96f482..d3488cd08c3 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -50,7 +50,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ steps.uv-cache.outputs.dir }} - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + key: ${{ runner.os }}-${{ matrix.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', matrix.python-version)) }} - name: Install dependencies run: make install-python-dependencies-ci - name: Test local integration tests diff --git a/Makefile b/Makefile index de2ee568b68..bef7437bc8a 100644 --- a/Makefile +++ b/Makefile @@ -268,7 +268,7 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests - test-python-universal-mysql-online: +test-python-universal-mysql-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.mysql_online_store.mysql_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ @@ -292,7 +292,11 @@ test-python-universal-cassandra: FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \ python -m pytest -x --integration \ - sdk/python/tests + sdk/python/tests/integration/offline_store/test_feature_logging.py \ + --ignore=sdk/python/tests/integration/offline_store/test_validation.py \ + -k "not test_snowflake and \ + not test_spark_materialization_consistency and \ + not test_universal_materialization" test-python-universal-hazelcast: PYTHONPATH='.' \ @@ -330,7 +334,7 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests - test-python-universal-elasticsearch-online: +test-python-universal-elasticsearch-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.elasticsearch \ @@ -349,6 +353,14 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests +test-python-universal-milvus-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_online_store.milvus_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus \ + python -m pytest -n 8 --integration \ + -k "test_retrieve_online_milvus_ocuments" \ + sdk/python/tests --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py + test-python-universal-singlestore-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.singlestore_repo_configuration \ diff --git a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md index 5e26f133cef..ee75aa6b74f 100644 --- a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md @@ -25,7 +25,7 @@ OnlineStore class names must end with the OnlineStore suffix! ### Contrib online stores -New online stores go in `sdk/python/feast/infra/online_stores/contrib/`. +New online stores go in `sdk/python/feast/infra/online_stores/`. #### What is a contrib plugin? diff --git a/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst index ee9faa55dc0..5ae3015bf37 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst @@ -4,6 +4,14 @@ feast.infra.online\_stores.milvus\_online\_store package Submodules ---------- +feast.infra.online\_stores.milvus\_online\_store.milvus module +-------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store.milvus + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.milvus\_online\_store.milvus\_repo\_configuration module ----------------------------------------------------------------------------------- diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py new file mode 100644 index 00000000000..a1a4a3a5fe5 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -0,0 +1,428 @@ +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union + +from pydantic import StrictStr +from pymilvus import ( + Collection, + CollectionSchema, + DataType, + FieldSchema, + connections, +) +from pymilvus.orm.connections import Connections + +from feast import Entity +from feast.feature_view import FeatureView +from feast.infra.infra_object import InfraObject +from feast.infra.key_encoding_utils import ( + serialize_entity_key, +) +from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig +from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.type_map import PROTO_VALUE_TO_VALUE_TYPE_MAP +from feast.types import ( + VALUE_TYPES_TO_FEAST_TYPES, + Array, + ComplexFeastType, + PrimitiveFeastType, + ValueType, +) +from feast.utils import ( + _build_retrieve_online_document_record, + _serialize_vector_to_float_list, + to_naive_utc, +) + +PROTO_TO_MILVUS_TYPE_MAPPING: Dict[ValueType, DataType] = { + PROTO_VALUE_TO_VALUE_TYPE_MAP["bytes_val"]: DataType.VARCHAR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["bool_val"]: DataType.BOOL, + PROTO_VALUE_TO_VALUE_TYPE_MAP["string_val"]: DataType.VARCHAR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["float_val"]: DataType.FLOAT, + PROTO_VALUE_TO_VALUE_TYPE_MAP["double_val"]: DataType.DOUBLE, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int32_val"]: DataType.INT32, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int64_val"]: DataType.INT64, + PROTO_VALUE_TO_VALUE_TYPE_MAP["float_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int32_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int64_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["double_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["bool_list_val"]: DataType.BINARY_VECTOR, +} + +FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING: Dict[ + Union[PrimitiveFeastType, Array, ComplexFeastType], DataType +] = {} + +for value_type, feast_type in VALUE_TYPES_TO_FEAST_TYPES.items(): + if isinstance(feast_type, PrimitiveFeastType): + milvus_type = PROTO_TO_MILVUS_TYPE_MAPPING.get(value_type) + if milvus_type: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = milvus_type + elif isinstance(feast_type, Array): + base_type = feast_type.base_type + base_value_type = base_type.to_value_type() + if base_value_type in [ + ValueType.INT32, + ValueType.INT64, + ValueType.FLOAT, + ValueType.DOUBLE, + ]: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.FLOAT_VECTOR + elif base_value_type == ValueType.STRING: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.VARCHAR + elif base_value_type == ValueType.BOOL: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.BINARY_VECTOR + + +class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): + """ + Configuration for the Milvus online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type: Literal["milvus"] = "milvus" + + host: Optional[StrictStr] = "localhost" + port: Optional[int] = 19530 + index_type: Optional[str] = "IVF_FLAT" + metric_type: Optional[str] = "L2" + embedding_dim: Optional[int] = 128 + vector_enabled: Optional[bool] = True + nlist: Optional[int] = 128 + + +class MilvusOnlineStore(OnlineStore): + """ + Milvus implementation of the online store interface. + + Attributes: + _collections: Dictionary to cache Milvus collections. + """ + + _conn: Optional[Connections] = None + _collections: Dict[str, Collection] = {} + + def _connect(self, config: RepoConfig) -> connections: + if not self._conn: + if not connections.has_connection("feast"): + self._conn = connections.connect( + alias="feast", + host=config.online_store.host, + port=str(config.online_store.port), + ) + return self._conn + + def _get_collection(self, config: RepoConfig, table: FeatureView) -> Collection: + collection_name = _table_id(config.project, table) + if collection_name not in self._collections: + self._connect(config) + + # Create a composite key by combining entity fields + composite_key_name = ( + "_".join([field.name for field in table.entity_columns]) + "_pk" + ) + + fields = [ + FieldSchema( + name=composite_key_name, + dtype=DataType.VARCHAR, + max_length=512, + is_primary=True, + ), + FieldSchema(name="event_ts", dtype=DataType.INT64), + FieldSchema(name="created_ts", dtype=DataType.INT64), + ] + fields_to_exclude = [ + "event_ts", + "created_ts", + ] + fields_to_add = [f for f in table.schema if f.name not in fields_to_exclude] + for field in fields_to_add: + dtype = FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING.get(field.dtype) + if dtype: + if dtype == DataType.FLOAT_VECTOR: + fields.append( + FieldSchema( + name=field.name, + dtype=dtype, + dim=config.online_store.embedding_dim, + ) + ) + elif dtype == DataType.VARCHAR: + fields.append( + FieldSchema( + name=field.name, + dtype=dtype, + max_length=512, + ) + ) + else: + fields.append(FieldSchema(name=field.name, dtype=dtype)) + + schema = CollectionSchema( + fields=fields, description="Feast feature view data" + ) + collection = Collection(name=collection_name, schema=schema, using="feast") + if not collection.has_index(): + index_params = { + "index_type": config.online_store.index_type, + "metric_type": config.online_store.metric_type, + "params": {"nlist": config.online_store.nlist}, + } + for vector_field in schema.fields: + if vector_field.dtype in [ + DataType.FLOAT_VECTOR, + DataType.BINARY_VECTOR, + ]: + collection.create_index( + field_name=vector_field.name, index_params=index_params + ) + collection.load() + self._collections[collection_name] = collection + return self._collections[collection_name] + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[ + EntityKeyProto, + Dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + collection = self._get_collection(config, table) + entity_batch_to_insert = [] + for entity_key, values_dict, timestamp, created_ts in data: + # need to construct the composite primary key also need to handle the fact that entities are a list + entity_key_str = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ).hex() + composite_key_name = ( + "_".join([str(value) for value in entity_key.join_keys]) + "_pk" + ) + timestamp_int = int(to_naive_utc(timestamp).timestamp() * 1e6) + created_ts_int = ( + int(to_naive_utc(created_ts).timestamp() * 1e6) if created_ts else 0 + ) + values_dict = _extract_proto_values_to_dict(values_dict) + entity_dict = _extract_proto_values_to_dict( + dict(zip(entity_key.join_keys, entity_key.entity_values)) + ) + values_dict.update(entity_dict) + + single_entity_record = { + composite_key_name: entity_key_str, + "event_ts": timestamp_int, + "created_ts": created_ts_int, + } + single_entity_record.update(values_dict) + entity_batch_to_insert.append(single_entity_record) + + if progress: + progress(1) + + collection.insert(entity_batch_to_insert) + collection.flush() + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + raise NotImplementedError + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + self._connect(config) + for table in tables_to_keep: + self._get_collection(config, table) + for table in tables_to_delete: + collection_name = _table_id(config.project, table) + collection = Collection(name=collection_name) + if collection.exists(): + collection.drop() + self._collections.pop(collection_name, None) + + def plan( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> List[InfraObject]: + raise NotImplementedError + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + self._connect(config) + for table in tables: + collection = self._get_collection(config, table) + if collection: + collection.drop() + self._collections.pop(collection.name, None) + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: Optional[str], + requested_features: Optional[List[str]], + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = None, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + collection = self._get_collection(config, table) + if not config.online_store.vector_enabled: + raise ValueError("Vector search is not enabled in the online store config") + + search_params = { + "metric_type": distance_metric or config.online_store.metric_type, + "params": {"nprobe": 10}, + } + expr = f"feature_name == '{requested_feature}'" + + composite_key_name = ( + "_".join([str(field.name) for field in table.entity_columns]) + "_pk" + ) + if requested_features: + features_str = ", ".join([f"'{f}'" for f in requested_features]) + expr += f" && feature_name in [{features_str}]" + + output_fields = ( + [composite_key_name] + + (requested_features if requested_features else []) + + ["created_ts", "event_ts"] + ) + assert all( + field + for field in output_fields + if field in [f.name for f in collection.schema.fields] + ), f"field(s) [{[field for field in output_fields if field not in [f.name for f in collection.schema.fields]]}'] not found in collection schema" + + # Note we choose the first vector field as the field to search on. Not ideal but it's something. + ann_search_field = None + for field in collection.schema.fields: + if ( + field.dtype in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR] + and field.name in output_fields + ): + ann_search_field = field.name + break + + results = collection.search( + data=[embedding], + anns_field=ann_search_field, + param=search_params, + limit=top_k, + output_fields=output_fields, + consistency_level="Strong", + ) + + result_list = [] + for hits in results: + for hit in hits: + single_record = {} + for field in output_fields: + single_record[field] = hit.entity.get(field) + + entity_key_bytes = bytes.fromhex(hit.entity.get(composite_key_name)) + embedding = hit.entity.get(ann_search_field) + serialized_embedding = _serialize_vector_to_float_list(embedding) + distance = hit.distance + event_ts = datetime.fromtimestamp(hit.entity.get("event_ts") / 1e6) + prepared_result = _build_retrieve_online_document_record( + entity_key_bytes, + # This may have a bug + serialized_embedding.SerializeToString(), + embedding, + distance, + event_ts, + config.entity_key_serialization_version, + ) + result_list.append(prepared_result) + return result_list + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _extract_proto_values_to_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]: + numeric_vector_list_types = [ + k + for k in PROTO_VALUE_TO_VALUE_TYPE_MAP.keys() + if k is not None and "list" in k and "string" not in k + ] + output_dict = {} + for feature_name, feature_values in input_dict.items(): + for proto_val_type in PROTO_VALUE_TO_VALUE_TYPE_MAP: + if feature_values.HasField(proto_val_type): + if proto_val_type in numeric_vector_list_types: + vector_values = getattr(feature_values, proto_val_type).val + else: + vector_values = getattr(feature_values, proto_val_type) + output_dict[feature_name] = vector_values + return output_dict + + +class MilvusTable(InfraObject): + """ + A Milvus collection managed by Feast. + + Attributes: + host: The host of the Milvus server. + port: The port of the Milvus server. + name: The name of the collection. + """ + + host: str + port: int + + def __init__(self, host: str, port: int, name: str): + super().__init__(name) + self.host = host + self.port = port + self._connect() + + def _connect(self): + return connections.connect(alias="default", host=self.host, port=str(self.port)) + + def to_infra_object_proto(self) -> InfraObjectProto: + # Implement serialization if needed + raise NotImplementedError + + def update(self): + # Implement update logic if needed + raise NotImplementedError + + def teardown(self): + collection = Collection(name=self.name) + if collection.exists(): + collection.drop() diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index fe34a12adf8..2b8d5174e1f 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -81,6 +81,7 @@ "singlestore": "feast.infra.online_stores.singlestore_online_store.singlestore.SingleStoreOnlineStore", "qdrant": "feast.infra.online_stores.cqdrant.QdrantOnlineStore", "couchbase": "feast.infra.online_stores.couchbase_online_store.couchbase.CouchbaseOnlineStore", + "milvus": "feast.infra.online_stores.milvus_online_store.milvus.MilvusOnlineStore", **LEGACY_ONLINE_STORE_CLASS_FOR_TYPE, } diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 8a88c24ffc1..000e9cdae4e 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -523,6 +523,24 @@ def python_values_to_proto_values( return proto_values +PROTO_VALUE_TO_VALUE_TYPE_MAP: Dict[str, ValueType] = { + "int32_val": ValueType.INT32, + "int64_val": ValueType.INT64, + "double_val": ValueType.DOUBLE, + "float_val": ValueType.FLOAT, + "string_val": ValueType.STRING, + "bytes_val": ValueType.BYTES, + "bool_val": ValueType.BOOL, + "int32_list_val": ValueType.INT32_LIST, + "int64_list_val": ValueType.INT64_LIST, + "double_list_val": ValueType.DOUBLE_LIST, + "float_list_val": ValueType.FLOAT_LIST, + "string_list_val": ValueType.STRING_LIST, + "bytes_list_val": ValueType.BYTES_LIST, + "bool_list_val": ValueType.BOOL_LIST, +} + + def _proto_value_to_value_type(proto_value: ProtoValue) -> ValueType: """ Returns Feast ValueType given Feast ValueType string. @@ -534,25 +552,9 @@ def _proto_value_to_value_type(proto_value: ProtoValue) -> ValueType: A variant of ValueType. """ proto_str = proto_value.WhichOneof("val") - type_map = { - "int32_val": ValueType.INT32, - "int64_val": ValueType.INT64, - "double_val": ValueType.DOUBLE, - "float_val": ValueType.FLOAT, - "string_val": ValueType.STRING, - "bytes_val": ValueType.BYTES, - "bool_val": ValueType.BOOL, - "int32_list_val": ValueType.INT32_LIST, - "int64_list_val": ValueType.INT64_LIST, - "double_list_val": ValueType.DOUBLE_LIST, - "float_list_val": ValueType.FLOAT_LIST, - "string_list_val": ValueType.STRING_LIST, - "bytes_list_val": ValueType.BYTES_LIST, - "bool_list_val": ValueType.BOOL_LIST, - None: ValueType.NULL, - } - - return type_map[proto_str] + if proto_str is None: + return ValueType.UNKNOWN + return PROTO_VALUE_TO_VALUE_TYPE_MAP[proto_str] def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType: diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 570a6d4f8d5..3d1f9219991 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -150,6 +150,7 @@ def retrieve_online_documents( config: RepoConfig, table: FeatureView, requested_feature: str, + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 4074dcb194e..d337d365e9b 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -614,6 +614,10 @@ def eventually_apply() -> Tuple[None, bool]: online_features = fs.get_online_features( features=features, entity_rows=entity_rows ).to_dict() + + # Debugging print statement + print("Online features values:", online_features["value"]) + assert all(v is None for v in online_features["value"]) @@ -891,3 +895,28 @@ def test_retrieve_online_documents(vectordb_environment, fake_document_data): top_k=2, distance_metric="wrong", ).to_dict() + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["milvus"]) +def test_retrieve_online_milvus_documents(vectordb_environment, fake_document_data): + fs = vectordb_environment.feature_store + df, data_source = fake_document_data + item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) + fs.apply([item_embeddings_feature_view, item()]) + fs.write_to_online_store("item_embeddings", df) + documents = fs.retrieve_online_documents( + feature=None, + features=[ + "item_embeddings:embedding_float", + "item_embeddings:item_id", + "item_embeddings:string_feature", + ], + query=[1.0, 2.0], + top_k=2, + distance_metric="L2", + ).to_dict() + assert len(documents["embedding_float"]) == 2 + + assert len(documents["item_id"]) == 2 + assert documents["item_id"] == [2, 3] From 3b6d8e373b67e37875958822aeb90ab4caa87cb1 Mon Sep 17 00:00:00 2001 From: xaniasd Date: Fri, 27 Dec 2024 20:34:17 +0100 Subject: [PATCH 60/90] chore: Update go targets in Makefile (#4861) * update go targets in Makefile Signed-off-by: Dimitris * Separate proto compilation for Go Signed-off-by: Dimitris --------- Signed-off-by: Dimitris --- .gitignore | 4 +++- Makefile | 60 ++++++++++++++++++++++++++++++++++++++---------------- setup.py | 58 ---------------------------------------------------- 3 files changed, 46 insertions(+), 76 deletions(-) diff --git a/.gitignore b/.gitignore index d558463c657..6823221aed4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ scratch* ### Local Environment ### *local*.env +tools ### Secret ### **/service_account.json @@ -101,6 +102,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +coverage.out *.cover .hypothesis/ .pytest_cache/ @@ -222,4 +224,4 @@ ui/.vercel **/yarn-error.log* # Go subprocess binaries (built during feast pip package building) -sdk/python/feast/binaries/ \ No newline at end of file +sdk/python/feast/binaries/ diff --git a/Makefile b/Makefile index bef7437bc8a..446e93eb1cf 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,13 @@ # limitations under the License. # -ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + +# install tools in project (tool) dir to not pollute the system +TOOL_DIR := $(ROOT_DIR)/tools +export GOBIN=$(TOOL_DIR)/bin +export PATH := $(TOOL_DIR)/bin:$(PATH) + MVN := mvn -f java/pom.xml ${MAVEN_EXTRA_OPTS} OS := linux ifeq ($(shell uname -s), Darwin) @@ -24,12 +30,15 @@ TRINO_VERSION ?= 376 PYTHON_VERSION = ${shell python --version | grep -Eo '[0-9]\.[0-9]+'} PYTHON_VERSIONS := 3.9 3.10 3.11 + define get_env_name $(subst .,,py$(1)) endef # General +$(TOOL_DIR): + mkdir -p $@/bin format: format-python format-java @@ -561,43 +570,60 @@ build-ui: # Go SDK & embedded -install-protoc-dependencies: - pip install "protobuf>=4.24.0,<5.0.0" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" +PB_REL = https://github.com/protocolbuffers/protobuf/releases +PB_VERSION = 3.11.2 +PB_ARCH := $(shell uname -m) +ifeq ($(PB_ARCH), arm64) + PB_ARCH=aarch_64 +endif +PB_PROTO_FOLDERS=core registry serving types storage + +$(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip: $(TOOL_DIR) + cd $(TOOL_DIR) && \ + curl -LO $(PB_REL)/download/v$(PB_VERSION)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip -install-go-proto-dependencies: +.PHONY: install-go-proto-dependencies +install-go-proto-dependencies: $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip + unzip -u $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip -d $(TOOL_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 +.PHONY: compile-protos-go +compile-protos-go: install-go-proto-dependencies + $(foreach folder,$(PB_PROTO_FOLDERS), \ + protoc --proto_path=$(ROOT_DIR)/protos \ + --go_out=$(ROOT_DIR)/go/protos \ + --go_opt=module=github.com/feast-dev/feast/go/protos \ + --go-grpc_out=$(ROOT_DIR)/go/protos \ + --go-grpc_opt=module=github.com/feast-dev/feast/go/protos $(ROOT_DIR)/protos/feast/$(folder)/*.proto; ) true + #install-go-ci-dependencies: # go install golang.org/x/tools/cmd/goimports # python -m pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" -build-go: - compile-protos-go +.PHONY: build-go +build-go: compile-protos-go go build -o feast ./go/main.go +.PHONY: install-feast-ci-locally install-feast-ci-locally: - pip install -e ".[ci]" + uv pip install -e ".[ci]" -test-go: - compile-protos-go - compile-protos-python - install-feast-ci-locally +.PHONY: test-go +test-go: compile-protos-go install-feast-ci-locally compile-protos-python CGO_ENABLED=1 go test -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html +.PHONY: format-go format-go: gofmt -s -w go/ -lint-go: - compile-protos-go +.PHONY: lint-go +lint-go: compile-protos-go go vet ./go/internal/feast +.PHONY: build-go-docker-dev build-go-docker-dev: docker buildx build --build-arg VERSION=dev \ -t feastdev/feature-server-go:dev \ -f go/infra/docker/feature-server/Dockerfile --load . -compile-protos-go: - install-go-proto-dependencies - install-protoc-dependencies - python setup.py build_go_protos \ No newline at end of file diff --git a/setup.py b/setup.py index 59b881c9715..034f6699105 100644 --- a/setup.py +++ b/setup.py @@ -257,61 +257,6 @@ PYTHON_CODE_PREFIX = "sdk/python" -def _generate_path_with_gopath(): - go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8") - go_path = go_path.strip() - path_val = os.getenv("PATH") - path_val = f"{path_val}:{go_path}/bin" - - return path_val - -class BuildGoProtosCommand(Command): - description = "Builds the proto files into Go files." - user_options = [] - - def initialize_options(self): - self.go_protoc = [ - sys.executable, - "-m", - "grpc_tools.protoc", - ] # find_executable("protoc") - self.proto_folder = os.path.join(repo_root, "protos") - self.go_folder = os.path.join(repo_root, "go/protos") - self.sub_folders = ["core", "registry", "serving", "types", "storage"] - self.path_val = _generate_path_with_gopath() - - def finalize_options(self): - pass - - def _generate_go_protos(self, path: str): - proto_files = glob.glob(os.path.join(self.proto_folder, path)) - - try: - subprocess.check_call( - self.go_protoc - + [ - "-I", - self.proto_folder, - "--go_out", - self.go_folder, - "--go_opt=module=github.com/feast-dev/feast/go/protos", - "--go-grpc_out", - self.go_folder, - "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos", - ] - + proto_files, - env={"PATH": self.path_val}, - ) - except CalledProcessError as e: - print(f"Stderr: {e.stderr}") - print(f"Stdout: {e.stdout}") - - def run(self): - go_dir = Path(repo_root) / "go" / "protos" - go_dir.mkdir(exist_ok=True) - for sub_folder in self.sub_folders: - self._generate_go_protos(f"feast/{sub_folder}/*.proto") - setup( name=NAME, @@ -376,7 +321,4 @@ def run(self): "pybindgen==0.22.0", # TODO do we need this? "setuptools_scm>=6.2", # TODO do we need this? ], - cmdclass={ - "build_go_protos": BuildGoProtosCommand - }, ) From 30ed749940ad1095c2f0c8db5cbeb083024e4bb0 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 27 Dec 2024 22:49:03 -0500 Subject: [PATCH 61/90] chore: Fixing testing in Milvus, Makefile, and repo_configuration (#4881) * chore: Fixing testing in Milvus, Makefile, and repo_configuration Signed-off-by: Francisco Javier Arceo * adjusted variable name Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * removed connection_string argument in config definition Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- Makefile | 2 +- .../tests/integration/feature_repos/repo_configuration.py | 5 +++++ .../integration/online_store/test_universal_online.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 446e93eb1cf..79628169218 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,7 @@ test-python-universal-milvus-online: FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_online_store.milvus_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus \ python -m pytest -n 8 --integration \ - -k "test_retrieve_online_milvus_ocuments" \ + -k "test_retrieve_online_milvus_documents" \ sdk/python/tests --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py test-python-universal-singlestore-online: diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index bf464681600..74bada19274 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -86,6 +86,7 @@ ) DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} +MILVUS_CONFIG = {"type": "milvus"} REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} REDIS_CLUSTER_CONFIG = { "type": "redis", @@ -162,6 +163,7 @@ AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) + AVAILABLE_ONLINE_STORES["milvus"] = (MILVUS_CONFIG, None) # Uncomment to test using private IKV account. Currently not enabled as # there is no dedicated IKV instance for CI testing and there is no @@ -559,6 +561,9 @@ def construct_test_environment( cache_ttl_seconds=1, ) + if test_repo_config.online_store in ["milvus", "pgvector", "qdrant"]: + entity_key_serialization_version = 3 + environment_params = { "name": project, "provider": test_repo_config.provider, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index d337d365e9b..64122d2c861 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -862,8 +862,8 @@ def assert_feature_service_entity_mapping_correctness( @pytest.mark.integration @pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch", "qdrant"]) -def test_retrieve_online_documents(vectordb_environment, fake_document_data): - fs = vectordb_environment.feature_store +def test_retrieve_online_documents(environment, fake_document_data): + fs = environment.feature_store df, data_source = fake_document_data item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) fs.apply([item_embeddings_feature_view, item()]) @@ -899,8 +899,8 @@ def test_retrieve_online_documents(vectordb_environment, fake_document_data): @pytest.mark.integration @pytest.mark.universal_online_stores(only=["milvus"]) -def test_retrieve_online_milvus_documents(vectordb_environment, fake_document_data): - fs = vectordb_environment.feature_store +def test_retrieve_online_milvus_documents(environment, fake_document_data): + fs = environment.feature_store df, data_source = fake_document_data item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) fs.apply([item_embeddings_feature_view, item()]) From ef724b66bd4d5f355b055d6d81525c4a17ce94c1 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Mon, 30 Dec 2024 05:01:13 -0600 Subject: [PATCH 62/90] feat: Add milvus package to release image & option to Operator (#4870) --- infra/feast-operator/api/v1alpha1/featurestore_types.go | 3 ++- .../config/crd/bases/feast.dev_featurestores.yaml | 2 ++ infra/feast-operator/dist/install.yaml | 2 ++ sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index f73c7fc6a40..f8d0c0f7da5 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -154,7 +154,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service type OnlineStoreDBStorePersistence struct { - // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase + // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase;milvus Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -178,6 +178,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "elasticsearch", "qdrant", "couchbase", + "milvus", } // LocalRegistryConfig configures the deployed registry service diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 2cab2d8c5d8..bd0f5aa61de 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -749,6 +749,7 @@ spec: - elasticsearch - qdrant - couchbase + - milvus type: string required: - secretRef @@ -2020,6 +2021,7 @@ spec: - elasticsearch - qdrant - couchbase + - milvus type: string required: - secretRef diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index cd63b3df8d0..ef2b1f2272c 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -757,6 +757,7 @@ spec: - elasticsearch - qdrant - couchbase + - milvus type: string required: - secretRef @@ -2028,6 +2029,7 @@ spec: - elasticsearch - qdrant - couchbase + - milvus type: string required: - secretRef diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index b4d7b5e3e9c..e6afb46aadf 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11-slim-bullseye RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb]" +RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus]" RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ From 170e2f08dde2c51f918f042e83a14fbe25307d41 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 3 Jan 2025 09:13:21 -0500 Subject: [PATCH 63/90] chore: Commenting out Milvus tests and removing from integration tests configuration (#4888) chore: Cmomenting out milvus tests and removing from integration configuration Signed-off-by: Francisco Javier Arceo --- .../feature_repos/repo_configuration.py | 2 +- .../online_store/test_universal_online.py | 46 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 74bada19274..54129f23c6e 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -163,7 +163,7 @@ AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) - AVAILABLE_ONLINE_STORES["milvus"] = (MILVUS_CONFIG, None) + # AVAILABLE_ONLINE_STORES["milvus"] = (MILVUS_CONFIG, None) # Uncomment to test using private IKV account. Currently not enabled as # there is no dedicated IKV instance for CI testing and there is no diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 64122d2c861..ab665914b5a 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -897,26 +897,26 @@ def test_retrieve_online_documents(environment, fake_document_data): ).to_dict() -@pytest.mark.integration -@pytest.mark.universal_online_stores(only=["milvus"]) -def test_retrieve_online_milvus_documents(environment, fake_document_data): - fs = environment.feature_store - df, data_source = fake_document_data - item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) - fs.apply([item_embeddings_feature_view, item()]) - fs.write_to_online_store("item_embeddings", df) - documents = fs.retrieve_online_documents( - feature=None, - features=[ - "item_embeddings:embedding_float", - "item_embeddings:item_id", - "item_embeddings:string_feature", - ], - query=[1.0, 2.0], - top_k=2, - distance_metric="L2", - ).to_dict() - assert len(documents["embedding_float"]) == 2 - - assert len(documents["item_id"]) == 2 - assert documents["item_id"] == [2, 3] +# @pytest.mark.integration +# @pytest.mark.universal_online_stores(only=["milvus"]) +# def test_retrieve_online_milvus_documents(environment, fake_document_data): +# fs = environment.feature_store +# df, data_source = fake_document_data +# item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) +# fs.apply([item_embeddings_feature_view, item()]) +# fs.write_to_online_store("item_embeddings", df) +# documents = fs.retrieve_online_documents( +# feature=None, +# features=[ +# "item_embeddings:embedding_float", +# "item_embeddings:item_id", +# "item_embeddings:string_feature", +# ], +# query=[1.0, 2.0], +# top_k=2, +# distance_metric="L2", +# ).to_dict() +# assert len(documents["embedding_float"]) == 2 +# +# assert len(documents["item_id"]) == 2 +# assert documents["item_id"] == [2, 3] From e04d7d51f97cd85e812f383019666adca0e80168 Mon Sep 17 00:00:00 2001 From: Gilad Leifman Date: Fri, 3 Jan 2025 16:32:54 +0200 Subject: [PATCH 64/90] =?UTF-8?q?chore:=20Minor=20language=20improvements?= =?UTF-8?q?=C2=A0=20(#4878)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Minor english updates Signed-off-by: Gilad Leifman --- docs/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/README.md b/docs/README.md index 36c83ed177a..73c6860137e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,12 +11,12 @@ for historical feature extraction used in model training and an (2) [online stor for serving features at low-latency in production systems and applications. Feast is a configurable operational data system that re-uses existing infrastructure to manage and serve machine learning -features to realtime models. For more details please review our [architecture](getting-started/architecture/overview.md). +features to realtime models. For more details, please review our [architecture](getting-started/architecture/overview.md). Concretely, Feast provides: -* A python SDK for programtically defining features, entities, sources, and (optionally) transformations -* A python SDK for for reading and writing features to configured offline and online data stores +* A Python SDK for programmatically defining features, entities, sources, and (optionally) transformations +* A Python SDK for reading and writing features to configured offline and online data stores * An [optional feature server](reference/feature-servers/README.md) for reading and writing features (useful for non-python languages) * A [UI](reference/alpha-web-ui.md) for viewing and exploring information about features defined in the project * A [CLI tool](reference/feast-cli-commands.md) for viewing and updating feature information @@ -24,8 +24,8 @@ Concretely, Feast provides: Feast allows ML platform teams to: * **Make features consistently available for training and low-latency serving** by managing an _offline store_ (to process historical data for scale-out batch scoring or model training), a low-latency _online store_ (to power real-time prediction)_,_ and a battle-tested _feature server_ (to serve pre-computed features online). -* **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training. -* **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another. +* **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensures that future feature values do not leak to models during training. +* **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to real-time models, and from one data infra system to another. {% hint style="info" %} **Note:** Feast today primarily addresses _timestamped structured data_. @@ -44,11 +44,11 @@ serving system must make a request to the feature store to retrieve feature valu Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams build a feature platform that improves collaboration between data engineers, software engineers, machine learning engineers, and data scientists. -* *For Data Scientists*: Feast is a a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. - +* *For Data Scientists*: Feast is a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. +    * *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. - -* *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. +    +* *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions, allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided Python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. * *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. @@ -56,14 +56,14 @@ Feast helps ML platform/MLOps teams with DevOps experience productionize real-ti ### Feast is not -* **an** [**ETL**](https://en.wikipedia.org/wiki/Extract,\_transform,\_load) / [**ELT**](https://en.wikipedia.org/wiki/Extract,\_load,\_transform) **system.** Feast is not a general purpose data pipelining system. Users often leverage tools like [dbt](https://www.getdbt.com) to manage upstream data transformations. Feast does support some [transformations](getting-started/architecture/feature-transformetion.md). -* **a data orchestration tool:** Feast does not manage or orchestrate complex workflow DAGs. It relies on upstream data pipelines to produce feature values and integrations with tools like [Airflow](https://airflow.apache.org) to make features consistently available. -* **a data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a light-weight downstream layer that can serve data from an existing data warehouse (or other data sources) to models in production. -* **a database:** Feast is not a database, but helps manage data stored in other systems (e.g. BigQuery, Snowflake, DynamoDB, Redis) to make features consistently available at training / serving time +* **An** [**ETL**](https://en.wikipedia.org/wiki/Extract,\_transform,\_load) / [**ELT**](https://en.wikipedia.org/wiki/Extract,\_load,\_transform) **system.** Feast is not a general purpose data pipelining system. Users often leverage tools like [dbt](https://www.getdbt.com) to manage upstream data transformations. Feast does support some [transformations](getting-started/architecture/feature-transformetion.md). +* **A data orchestration tool:** Feast does not manage or orchestrate complex workflow DAGs. It relies on upstream data pipelines to produce feature values and integrations with tools like [Airflow](https://airflow.apache.org) to make features consistently available. +* **A data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a lightweight downstream layer that can serve data from an existing data warehouse (or other data sources) to models in production. +* **A database:** Feast is not a database, but helps manage data stored in other systems (e.g. BigQuery, Snowflake, DynamoDB, Redis) to make features consistently available at training / serving time ### Feast does not _fully_ solve * **reproducible model training / model backtesting / experiment management**: Feast captures feature and model metadata, but does not version-control datasets / labels or manage train / test splits. Other tools like [DVC](https://dvc.org/), [MLflow](https://www.mlflow.org/), and [Kubeflow](https://www.kubeflow.org/) are better suited for this. -* **batch feature engineering**: Feast supports on demand and streaming transformations. Feast is also investing in supporting batch transformations. +* **batch feature engineering**: Feast supports on-demand and streaming transformations. Feast is also investing in supporting batch transformations. * **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. * **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). * **data quality / drift detection**: Feast has experimental integrations with [Great Expectations](https://greatexpectations.io/), but is not purpose built to solve data drift / data quality issues. This requires more sophisticated monitoring across data pipelines, served feature values, labels, and model versions. @@ -75,7 +75,7 @@ Many companies have used Feast to power real-world ML use cases such as: * Personalizing online recommendations by leveraging pre-computed historical user or item features. * Online fraud detection, using features that compare against (pre-computed) historical transaction patterns * Churn prediction (an offline model), generating feature values for all users at a fixed cadence in batch -* Credit scoring, using pre-computed historical features to compute probability of default +* Credit scoring, using pre-computed historical features to compute the probability of default ## How can I get started? From c62377bc095a83022d13e5a8a3a9413d7e0f3e2c Mon Sep 17 00:00:00 2001 From: xaniasd Date: Fri, 3 Jan 2025 16:34:55 +0100 Subject: [PATCH 65/90] fix: Make transformation_service_endpoint configuration optional (#4880) * Make transformation_service_endpoint configuration optional Signed-off-by: Dimitris Stafylarakis * Add custom error for transformation service, implement featurestore unit tests Signed-off-by: Dimitris Stafylarakis --------- Signed-off-by: Dimitris Stafylarakis --- go/internal/feast/errors.go | 22 ++ go/internal/feast/featurestore.go | 22 +- go/internal/feast/featurestore_test.go | 277 ++++++++++++++++------- go/internal/test/feature_repo/example.py | 36 ++- 4 files changed, 264 insertions(+), 93 deletions(-) create mode 100644 go/internal/feast/errors.go diff --git a/go/internal/feast/errors.go b/go/internal/feast/errors.go new file mode 100644 index 00000000000..f42b4aad82d --- /dev/null +++ b/go/internal/feast/errors.go @@ -0,0 +1,22 @@ +package feast + +import ( + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type FeastTransformationServiceNotConfigured struct{} + +func (FeastTransformationServiceNotConfigured) GRPCStatus() *status.Status { + errorStatus := status.New(codes.Internal, "No transformation service configured") + ds, err := errorStatus.WithDetails(&errdetails.LocalizedMessage{Message: "No transformation service configured, required for on-demand feature transformations"}) + if err != nil { + return errorStatus + } + return ds +} + +func (e FeastTransformationServiceNotConfigured) Error() string { + return e.GRPCStatus().Err().Error() +} diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index df4df7e1995..abe1d195def 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -3,8 +3,9 @@ package feast import ( "context" "errors" - "fmt" + "github.com/apache/arrow/go/v17/arrow/memory" + //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/feast-dev/feast/go/internal/feast/model" @@ -60,17 +61,14 @@ func NewFeatureStore(config *registry.RepoConfig, callback transformation.Transf return nil, err } - // Use a scalable transformation service like Python Transformation Service. - // Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file - // under the "feature_server" section. - transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"] - if !ok { - fmt.Println("Errors while reading transformation_service_endpoint info") - panic("No transformation service endpoint provided in the feature_store.yaml file.") + var transformationService *transformation.GrpcTransformationService + if transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"]; ok { + // Use a scalable transformation service like Python Transformation Service. + // Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file + // under the "feature_server" section. + transformationService, _ = transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string)) } - transformationService, _ := transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string)) - return &FeatureStore{ config: config, registry: registry, @@ -112,6 +110,10 @@ func (fs *FeatureStore) GetOnlineFeatures( return nil, err } + if len(requestedOnDemandFeatureViews) > 0 && fs.transformationService == nil { + return nil, FeastTransformationServiceNotConfigured{} + } + entityNameToJoinKeyMap, expectedJoinKeysSet, err := onlineserving.GetEntityMaps(requestedFeatureViews, entities) if err != nil { return nil, err diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index f066b39df2d..e1f908b9062 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -2,124 +2,241 @@ package feast import ( "context" + "log" + "os" "path/filepath" "runtime" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/feast-dev/feast/go/internal/feast/onlinestore" "github.com/feast-dev/feast/go/internal/feast/registry" + "github.com/feast-dev/feast/go/internal/test" + "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" ) -// Return absolute path to the test_repo registry regardless of the working directory -func getRegistryPath() map[string]interface{} { +var featureRepoBasePath string +var featureRepoRegistryFile string + +func TestMain(m *testing.M) { // Get the file path of this source file, regardless of the working directory _, filename, _, ok := runtime.Caller(0) if !ok { - panic("couldn't find file path of the test file") + log.Print("couldn't find file path of the test file") + os.Exit(1) } - registry := map[string]interface{}{ - "path": filepath.Join(filename, "..", "..", "..", "feature_repo/data/registry.db"), + featureRepoBasePath = filepath.Join(filename, "..", "..", "test") + featureRepoRegistryFile = filepath.Join(featureRepoBasePath, "feature_repo", "data", "registry.db") + if err := test.SetupInitializedRepo(featureRepoBasePath); err != nil { + log.Print("Could not initialize test repo: ", err) + os.Exit(1) } - return registry + os.Exit(m.Run()) } func TestNewFeatureStore(t *testing.T) { - t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := registry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", - }, - } - fs, err := NewFeatureStore(&config, nil) - assert.Nil(t, err) - assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore) - - t.Run("valid config", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", + tests := []struct { + name string + config *registry.RepoConfig + expectOnlineStoreType interface{} + errMessage string + }{ + { + name: "valid config", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + }, }, - FeatureServer: map[string]interface{}{ - "transformation_service_endpoint": "localhost:50051", + expectOnlineStoreType: &onlinestore.RedisOnlineStore{}, + }, + { + name: "valid config with transformation service endpoint", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + }, + FeatureServer: map[string]interface{}{ + "transformation_service_endpoint": "localhost:50051", + }, }, - } - fs, err := NewFeatureStore(config, nil) - assert.Nil(t, err) - assert.NotNil(t, fs) - assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore) - assert.NotNil(t, fs.transformationService) - }) - - t.Run("missing transformation service endpoint", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", + expectOnlineStoreType: &onlinestore.RedisOnlineStore{}, + }, + { + name: "invalid online store config", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "invalid_store", + }, }, - } - defer func() { - if r := recover(); r == nil { - t.Errorf("The code did not panic") + errMessage: "invalid_store online store type is currently not supported", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := NewFeatureStore(test.config, nil) + if test.errMessage != "" { + assert.Nil(t, got) + require.Error(t, err) + assert.ErrorContains(t, err, test.errMessage) + + } else { + require.NoError(t, err) + assert.NotNil(t, got) + assert.IsType(t, test.expectOnlineStoreType, got.onlineStore) } - }() - NewFeatureStore(config, nil) - }) - - t.Run("invalid online store config", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "invalid_store", + }) + } + +} + +type MockRedis struct { + mock.Mock +} + +func (m *MockRedis) Destruct() {} +func (m *MockRedis) OnlineRead(ctx context.Context, entityKeys []*types.EntityKey, featureViewNames []string, featureNames []string) ([][]onlinestore.FeatureData, error) { + args := m.Called(ctx, entityKeys, featureViewNames, featureNames) + var fd [][]onlinestore.FeatureData + if args.Get(0) != nil { + fd = args.Get(0).([][]onlinestore.FeatureData) + } + return fd, args.Error(1) +} + +func TestGetOnlineFeatures(t *testing.T) { + tests := []struct { + name string + config *registry.RepoConfig + fn func(*testing.T, *FeatureStore) + }{ + { + name: "redis with simple features", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + "connection_string": "localhost:6379", + }, }, - FeatureServer: map[string]interface{}{ - "transformation_service_endpoint": "localhost:50051", + fn: testRedisSimpleFeatures, + }, + { + name: "redis with On-demand feature views, no transformation service endpoint", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + "connection_string": "localhost:6379", + }, }, - } - fs, err := NewFeatureStore(config, nil) - assert.NotNil(t, err) - assert.Nil(t, fs) - }) + fn: testRedisODFVNoTransformationService, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fs, err := NewFeatureStore(test.config, nil) + require.Nil(t, err) + fs.onlineStore = new(MockRedis) + test.fn(t, fs) + }) + + } } -func TestGetOnlineFeaturesRedis(t *testing.T) { - t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := registry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", - "connection_string": "localhost:6379", +func testRedisSimpleFeatures(t *testing.T, fs *FeatureStore) { + + featureNames := []string{"driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + } + entities := map[string]*types.RepeatedValue{"driver_id": {Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1002}}, + }}} + + results := [][]onlinestore.FeatureData{ + { + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "conv_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 12.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "acc_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 1.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "avg_daily_trips"}, + Value: types.Value{Val: &types.Value_Int64Val{Int64Val: 100}}, + }, + }, + { + + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "conv_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 24.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "acc_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 2.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "avg_daily_trips"}, + Value: types.Value{Val: &types.Value_Int64Val{Int64Val: 130}}, + }, }, } + ctx := context.Background() + mr := fs.onlineStore.(*MockRedis) + mr.On("OnlineRead", ctx, mock.Anything, mock.Anything, mock.Anything).Return(results, nil) + response, err := fs.GetOnlineFeatures(ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) + require.Nil(t, err) + assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response +} +func testRedisODFVNoTransformationService(t *testing.T, fs *FeatureStore) { featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", } entities := map[string]*types.RepeatedValue{"driver_id": {Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, {Val: &types.Value_Int64Val{Int64Val: 1002}}, {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, } - fs, err := NewFeatureStore(&config, nil) - assert.Nil(t, err) ctx := context.Background() - response, err := fs.GetOnlineFeatures( - ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) - assert.Nil(t, err) - assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response + mr := fs.onlineStore.(*MockRedis) + mr.On("OnlineRead", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + response, err := fs.GetOnlineFeatures(ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) + assert.Nil(t, response) + assert.ErrorAs(t, err, &FeastTransformationServiceNotConfigured{}) + } diff --git a/go/internal/test/feature_repo/example.py b/go/internal/test/feature_repo/example.py index 70843610075..a814b58913b 100644 --- a/go/internal/test/feature_repo/example.py +++ b/go/internal/test/feature_repo/example.py @@ -2,10 +2,12 @@ from datetime import timedelta -from feast import Entity, Feature, FeatureView, Field, FileSource, FeatureService +from feast import Entity, Feature, FeatureView, Field, FileSource, FeatureService, RequestSource from feast.feature_logging import LoggingConfig from feast.infra.offline_stores.file_source import FileLoggingDestination -from feast.types import Float32, Int64 +from feast.types import Float32, Float64, Int64, PrimitiveFeastType +from feast.on_demand_feature_view import on_demand_feature_view +import pandas as pd # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation @@ -41,4 +43,32 @@ name="test_service", features=[driver_hourly_stats_view], logging_config=LoggingConfig(destination=FileLoggingDestination(path="")) -) \ No newline at end of file +) + + +# Define a request data source which encodes features / information only +# available at request time (e.g. part of the user initiated HTTP request) +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + ] +) + +# Use the input data and feature view features to create new features +@on_demand_feature_view( + sources=[ + driver_hourly_stats_view, + input_request + ], + schema=[ + Field(name='conv_rate_plus_val1', dtype=Float64), + Field(name='conv_rate_plus_val2', dtype=Float64) + ] +) +def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df['conv_rate_plus_val1'] = (features_df['conv_rate'] + features_df['val_to_add']) + df['conv_rate_plus_val2'] = (features_df['conv_rate'] + features_df['val_to_add_2']) + return df From ca45a1c3cc2fceca2494b4323c011a135097e4c1 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 3 Jan 2025 14:52:15 -0500 Subject: [PATCH 66/90] chore: Add Milvus to pr_integration_tests.yml (#4891) * Add Milvus to pr_integration_tests.yml * upgrading to v2.5.1 and removing milvus service key --- .github/workflows/pr_integration_tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 923c0b0335b..edf92e7e786 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -95,6 +95,10 @@ jobs: run: | docker pull vishnunair/docker-redis-cluster:latest docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster + - name: Setup Milvus Cluster + run: | + wget https://github.com/milvus-io/milvus/releases/download/v2.5.1/milvus-standalone-docker-compose.yml -O docker-compose.yml + docker compose up -d - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: From 07958f71cd89984325ec3ca2006b17fe5d333d02 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 6 Jan 2025 11:15:30 -0500 Subject: [PATCH 67/90] revert: Revert "chore: Add Milvus to pr_integration_tests.yml" (#4900) Revert "chore: Add Milvus to pr_integration_tests.yml (#4891)" This reverts commit ca45a1c3cc2fceca2494b4323c011a135097e4c1. --- .github/workflows/pr_integration_tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index edf92e7e786..923c0b0335b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -95,10 +95,6 @@ jobs: run: | docker pull vishnunair/docker-redis-cluster:latest docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - - name: Setup Milvus Cluster - run: | - wget https://github.com/milvus-io/milvus/releases/download/v2.5.1/milvus-standalone-docker-compose.yml -O docker-compose.yml - docker compose up -d - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: From 76e1e2178c285886136e8f2fc4436302e4291715 Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:44:26 -0500 Subject: [PATCH 68/90] feat: Separating the RBAC and Remote related integration tests. (#4905) * Separating the tests related to remote and rbac functionality. * Added a new test marker to separate the tests related to rbac and remote functionality. * Added a new github action to perform tests related to rbac and remote functionality. * Filtered the rbac integration tests in the current github job. and added new make target to run the new tests. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../operator-e2e-integration-tests.yml | 4 ++ .../pr_remote_rbac_integration_tests.yml | 58 +++++++++++++++++++ Makefile | 13 +++++ sdk/python/pytest.ini | 1 + sdk/python/tests/conftest.py | 4 ++ sdk/python/tests/integration/conftest.py | 21 ++++++- .../universal/data_source_creator.py | 9 +++ .../universal/data_sources/file.py | 14 +++++ .../online_store/test_remote_online_store.py | 1 + .../registration/test_universal_registry.py | 5 +- 10 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pr_remote_rbac_integration_tests.yml diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index cbb505c3fe8..a06e793410e 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -10,6 +10,10 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: operator-e2e-tests: diff --git a/.github/workflows/pr_remote_rbac_integration_tests.yml b/.github/workflows/pr_remote_rbac_integration_tests.yml new file mode 100644 index 00000000000..98fa5a52c58 --- /dev/null +++ b/.github/workflows/pr_remote_rbac_integration_tests.yml @@ -0,0 +1,58 @@ +name: pr-remote-rbac-integration-tests +# This runs the integration tests related to rbac functionality and remote registry and online features. + +on: + pull_request: + types: + - opened + - synchronize + - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' + +jobs: + remote-rbac-integration-tests-python: + if: + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.event.pull_request.base.repo.full_name == 'feast-dev/feast' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.11" ] + os: [ ubuntu-latest ] + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ github.event.repository.full_name }} # Uses the full repository name + ref: ${{ github.ref }} # Uses the ref from the event + token: ${{ secrets.GITHUB_TOKEN }} # Automatically provided token + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v5 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache + run: | + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ matrix.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', matrix.python-version)) }} + - name: Install dependencies + run: make install-python-dependencies-ci + - name: Test rbac and remote feature integration tests + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + run: make test-python-integration-rbac-remote diff --git a/Makefile b/Makefile index 79628169218..0d08f002bb3 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,8 @@ test-python-unit: test-python-integration: python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ + -m "not rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-local: @@ -114,6 +116,17 @@ test-python-integration-local: FEAST_LOCAL_ONLINE_CONTAINER=True \ python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ + -m "not rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ + sdk/python/tests + +test-python-integration-rbac-remote: + FEAST_IS_LOCAL_TEST=True \ + FEAST_LOCAL_ONLINE_CONTAINER=True \ + python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + -k "not test_lambda_materialization and not test_snowflake_materialization" \ + -m "rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-container: diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index a0736767601..d79459c0d0e 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -4,6 +4,7 @@ asyncio_mode = auto markers = universal_offline_stores: mark a test as using all offline stores. universal_online_stores: mark a test as using all online stores. + rbac_remote_integration_test: mark a integration test related to rbac and remote functionality. env = IS_TEST=True diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 6e5f1e14870..c029648aeeb 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -310,6 +310,10 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): pytest.mark.xdist_group(name=m) for m in c.offline_store_creator.xdist_groups() ] + # Check if there are any test markers associated with the creator and add them. + if c.offline_store_creator.test_markers(): + marks.extend(c.offline_store_creator.test_markers()) + _config_cache[c] = pytest.param(c, marks=marks) configs.append(_config_cache[c]) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 82f80b89927..21c9051d0d7 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -1,4 +1,7 @@ import logging +import random +import time +from multiprocessing import Manager import pytest from testcontainers.keycloak import KeycloakContainer @@ -9,14 +12,30 @@ from tests.utils.auth_permissions_util import setup_permissions_on_keycloak logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +shared_state = Manager().dict() @pytest.fixture(scope="session") def start_keycloak_server(): + # Add random sleep between 0 and 2 before checking the state to avoid concurrency issues. + random_sleep_time = random.uniform(0, 2) + time.sleep(random_sleep_time) + + # If the Keycloak instance is already started (in any worker), reuse it + if shared_state.get("keycloak_started", False): + return shared_state["keycloak_url"] logger.info("Starting keycloak instance") with KeycloakContainer("quay.io/keycloak/keycloak:24.0.1") as keycloak_container: setup_permissions_on_keycloak(keycloak_container.get_client()) - yield keycloak_container.get_url() + shared_state["keycloak_started"] = True + shared_state["keycloak_url"] = keycloak_container.get_url() + yield shared_state["keycloak_url"] + + # After the fixture is done, cleanup the shared state + del shared_state["keycloak_started"] + del shared_state["keycloak_url"] @pytest.fixture(scope="session") diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index 513a94ee210..467db4dddce 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -2,6 +2,7 @@ from typing import Dict, Optional import pandas as pd +from _pytest.mark import MarkDecorator from feast.data_source import DataSource from feast.feature_logging import LoggingDestination @@ -64,3 +65,11 @@ def teardown(self): @staticmethod def xdist_groups() -> list[str]: return [] + + @staticmethod + def test_markers() -> list[MarkDecorator]: + """ + return the array of test markers to add dynamically to the tests created by this creator method. override this method in your implementations. By default, it will not add any markers. + :return: + """ + return [] diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 1d33402e012..6f6e5d68133 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -11,7 +11,9 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq +import pytest import yaml +from _pytest.mark import MarkDecorator from minio import Minio from testcontainers.core.generic import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -372,6 +374,10 @@ def __init__(self, project_name: str, *args, **kwargs): self.server_port: int = 0 self.proc: Optional[Popen[bytes]] = None + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( @@ -418,6 +424,10 @@ def __init__(self, project_name: str, *args, **kwargs): self.server_port: int = 0 self.proc: Optional[Popen[bytes]] = None + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( @@ -515,6 +525,10 @@ def __init__(self, project_name: str, *args, **kwargs): def xdist_groups() -> list[str]: return ["keycloak"] + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 285253dfaaf..eb03fd0c3c5 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -22,6 +22,7 @@ @pytest.mark.integration +@pytest.mark.rbac_remote_integration_test @pytest.mark.parametrize( "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True ) diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 5e06247ebbb..3819d168d78 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -344,7 +344,10 @@ def mock_remote_registry(): marks=pytest.mark.xdist_group(name="mysql_registry"), ), lazy_fixture("sqlite_registry"), - lazy_fixture("mock_remote_registry"), + pytest.param( + lazy_fixture("mock_remote_registry"), + marks=pytest.mark.rbac_remote_integration_test, + ), ] sql_fixtures = [ From 5f9b5b54868ab31be6ad1570136d5de38927584a Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 8 Jan 2025 16:14:43 -0500 Subject: [PATCH 69/90] chore: Moving Milvus client to PyMilvus (#4907) * chore: Moving Milvus client to PyMilvus Signed-off-by: Francisco Javier Arceo * linted and switched implementation to pymilvus Signed-off-by: Francisco Javier Arceo * adding updates for integration configuration Signed-off-by: Francisco Javier Arceo * removing drop statement Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 2 +- .../milvus_online_store/milvus.py | 147 ++++++++++-------- .../universal/online_store/milvus.py | 12 +- .../online_store/test_universal_online.py | 46 +++--- 4 files changed, 118 insertions(+), 89 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 4564d6abf3c..98db710d7f3 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1757,7 +1757,7 @@ def retrieve_online_documents( query: Union[str, List[float]], top_k: int, features: Optional[List[str]] = None, - distance_metric: Optional[str] = None, + distance_metric: Optional[str] = "L2", ) -> OnlineResponse: """ Retrieves the top k closest document features. Note, embeddings are a subset of features. diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index a1a4a3a5fe5..8d5405c428a 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -7,9 +7,8 @@ CollectionSchema, DataType, FieldSchema, - connections, + MilvusClient, ) -from pymilvus.orm.connections import Connections from feast import Entity from feast.feature_view import FeatureView @@ -85,7 +84,6 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """ type: Literal["milvus"] = "milvus" - host: Optional[StrictStr] = "localhost" port: Optional[int] = 19530 index_type: Optional[str] = "IVF_FLAT" @@ -93,6 +91,8 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): embedding_dim: Optional[int] = 128 vector_enabled: Optional[bool] = True nlist: Optional[int] = 128 + username: Optional[StrictStr] = "" + password: Optional[StrictStr] = "" class MilvusOnlineStore(OnlineStore): @@ -103,24 +103,23 @@ class MilvusOnlineStore(OnlineStore): _collections: Dictionary to cache Milvus collections. """ - _conn: Optional[Connections] = None - _collections: Dict[str, Collection] = {} + client: Optional[MilvusClient] = None + _collections: Dict[str, Any] = {} - def _connect(self, config: RepoConfig) -> connections: - if not self._conn: - if not connections.has_connection("feast"): - self._conn = connections.connect( - alias="feast", - host=config.online_store.host, - port=str(config.online_store.port), - ) - return self._conn + def _connect(self, config: RepoConfig) -> MilvusClient: + if not self.client: + self.client = MilvusClient( + url=f"{config.online_store.host}:{config.online_store.port}", + token=f"{config.online_store.username}:{config.online_store.password}" + if config.online_store.username and config.online_store.password + else "", + ) + return self.client - def _get_collection(self, config: RepoConfig, table: FeatureView) -> Collection: + def _get_collection(self, config: RepoConfig, table: FeatureView) -> Dict[str, Any]: + self.client = self._connect(config) collection_name = _table_id(config.project, table) if collection_name not in self._collections: - self._connect(config) - # Create a composite key by combining entity fields composite_key_name = ( "_".join([field.name for field in table.entity_columns]) + "_pk" @@ -166,23 +165,38 @@ def _get_collection(self, config: RepoConfig, table: FeatureView) -> Collection: schema = CollectionSchema( fields=fields, description="Feast feature view data" ) - collection = Collection(name=collection_name, schema=schema, using="feast") - if not collection.has_index(): - index_params = { - "index_type": config.online_store.index_type, - "metric_type": config.online_store.metric_type, - "params": {"nlist": config.online_store.nlist}, - } - for vector_field in schema.fields: - if vector_field.dtype in [ - DataType.FLOAT_VECTOR, - DataType.BINARY_VECTOR, - ]: - collection.create_index( - field_name=vector_field.name, index_params=index_params - ) - collection.load() - self._collections[collection_name] = collection + collection_exists = self.client.has_collection( + collection_name=collection_name + ) + if not collection_exists: + self.client.create_collection( + collection_name=collection_name, + dimension=config.online_store.embedding_dim, + schema=schema, + ) + index_params = self.client.prepare_index_params() + for vector_field in schema.fields: + if vector_field.dtype in [ + DataType.FLOAT_VECTOR, + DataType.BINARY_VECTOR, + ]: + index_params.add_index( + collection_name=collection_name, + field_name=vector_field.name, + metric_type=config.online_store.metric_type, + index_type=config.online_store.index_type, + index_name=f"vector_index_{vector_field.name}", + params={"nlist": config.online_store.nlist}, + ) + self.client.create_index( + collection_name=collection_name, + index_params=index_params, + ) + else: + self.client.load_collection(collection_name) + self._collections[collection_name] = self.client.describe_collection( + collection_name + ) return self._collections[collection_name] def online_write_batch( @@ -199,6 +213,7 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: + self.client = self._connect(config) collection = self._get_collection(config, table) entity_batch_to_insert = [] for entity_key, values_dict, timestamp, created_ts in data: @@ -231,8 +246,9 @@ def online_write_batch( if progress: progress(1) - collection.insert(entity_batch_to_insert) - collection.flush() + self.client.insert( + collection_name=collection["collection_name"], data=entity_batch_to_insert + ) def online_read( self, @@ -252,14 +268,14 @@ def update( entities_to_keep: Sequence[Entity], partial: bool, ): - self._connect(config) + self.client = self._connect(config) for table in tables_to_keep: - self._get_collection(config, table) + self._collections = self._get_collection(config, table) + for table in tables_to_delete: collection_name = _table_id(config.project, table) - collection = Collection(name=collection_name) - if collection.exists(): - collection.drop() + if self._collections.get(collection_name, None): + self.client.drop_collection(collection_name) self._collections.pop(collection_name, None) def plan( @@ -273,12 +289,12 @@ def teardown( tables: Sequence[FeatureView], entities: Sequence[Entity], ): - self._connect(config) + self.client = self._connect(config) for table in tables: - collection = self._get_collection(config, table) - if collection: - collection.drop() - self._collections.pop(collection.name, None) + collection_name = _table_id(config.project, table) + if self._collections.get(collection_name, None): + self.client.drop_collection(collection_name) + self._collections.pop(collection_name, None) def retrieve_online_documents( self, @@ -298,6 +314,8 @@ def retrieve_online_documents( Optional[ValueProto], ] ]: + self.client = self._connect(config) + collection_name = _table_id(config.project, table) collection = self._get_collection(config, table) if not config.online_store.vector_enabled: raise ValueError("Vector search is not enabled in the online store config") @@ -321,28 +339,27 @@ def retrieve_online_documents( + ["created_ts", "event_ts"] ) assert all( - field + field in [f["name"] for f in collection["fields"]] for field in output_fields - if field in [f.name for f in collection.schema.fields] - ), f"field(s) [{[field for field in output_fields if field not in [f.name for f in collection.schema.fields]]}'] not found in collection schema" - + ), f"field(s) [{[field for field in output_fields if field not in [f['name'] for f in collection['fields']]]}] not found in collection schema" # Note we choose the first vector field as the field to search on. Not ideal but it's something. ann_search_field = None - for field in collection.schema.fields: + for field in collection["fields"]: if ( - field.dtype in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR] - and field.name in output_fields + field["type"] in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR] + and field["name"] in output_fields ): - ann_search_field = field.name + ann_search_field = field["name"] break - results = collection.search( + self.client.load_collection(collection_name) + results = self.client.search( + collection_name=collection_name, data=[embedding], anns_field=ann_search_field, - param=search_params, + search_params=search_params, limit=top_k, output_fields=output_fields, - consistency_level="Strong", ) result_list = [] @@ -350,13 +367,17 @@ def retrieve_online_documents( for hit in hits: single_record = {} for field in output_fields: - single_record[field] = hit.entity.get(field) + single_record[field] = hit.get("entity", {}).get(field, None) - entity_key_bytes = bytes.fromhex(hit.entity.get(composite_key_name)) - embedding = hit.entity.get(ann_search_field) + entity_key_bytes = bytes.fromhex( + hit.get("entity", {}).get(composite_key_name, None) + ) + embedding = hit.get("entity", {}).get(ann_search_field) serialized_embedding = _serialize_vector_to_float_list(embedding) - distance = hit.distance - event_ts = datetime.fromtimestamp(hit.entity.get("event_ts") / 1e6) + distance = hit.get("distance", None) + event_ts = datetime.fromtimestamp( + hit.get("entity", {}).get("event_ts") / 1e6 + ) prepared_result = _build_retrieve_online_document_record( entity_key_bytes, # This may have a bug @@ -412,7 +433,7 @@ def __init__(self, host: str, port: int, name: str): self._connect() def _connect(self): - return connections.connect(alias="default", host=self.host, port=str(self.port)) + raise NotImplementedError def to_infra_object_proto(self) -> InfraObjectProto: # Implement serialization if needed diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py index 8ffee04c12f..c02bd144016 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -1,6 +1,8 @@ from typing import Any, Dict -from testcontainers.milvus import MilvusContainer +import docker +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs from tests.integration.feature_repos.universal.online_store_creator import ( OnlineStoreCreator, @@ -11,13 +13,19 @@ class MilvusOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) self.fixed_port = 19530 - self.container = MilvusContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( + self.container = DockerContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( self.fixed_port ) + self.client = docker.from_env() def create_online_store(self) -> Dict[str, Any]: self.container.start() # Wait for Milvus server to be ready + # log_string_to_wait_for = "Ready to accept connections" + log_string_to_wait_for = "" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=30 + ) host = "localhost" port = self.container.get_exposed_port(self.fixed_port) return { diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index ab665914b5a..64122d2c861 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -897,26 +897,26 @@ def test_retrieve_online_documents(environment, fake_document_data): ).to_dict() -# @pytest.mark.integration -# @pytest.mark.universal_online_stores(only=["milvus"]) -# def test_retrieve_online_milvus_documents(environment, fake_document_data): -# fs = environment.feature_store -# df, data_source = fake_document_data -# item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) -# fs.apply([item_embeddings_feature_view, item()]) -# fs.write_to_online_store("item_embeddings", df) -# documents = fs.retrieve_online_documents( -# feature=None, -# features=[ -# "item_embeddings:embedding_float", -# "item_embeddings:item_id", -# "item_embeddings:string_feature", -# ], -# query=[1.0, 2.0], -# top_k=2, -# distance_metric="L2", -# ).to_dict() -# assert len(documents["embedding_float"]) == 2 -# -# assert len(documents["item_id"]) == 2 -# assert documents["item_id"] == [2, 3] +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["milvus"]) +def test_retrieve_online_milvus_documents(environment, fake_document_data): + fs = environment.feature_store + df, data_source = fake_document_data + item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) + fs.apply([item_embeddings_feature_view, item()]) + fs.write_to_online_store("item_embeddings", df) + documents = fs.retrieve_online_documents( + feature=None, + features=[ + "item_embeddings:embedding_float", + "item_embeddings:item_id", + "item_embeddings:string_feature", + ], + query=[1.0, 2.0], + top_k=2, + distance_metric="L2", + ).to_dict() + assert len(documents["embedding_float"]) == 2 + + assert len(documents["item_id"]) == 2 + assert documents["item_id"] == [2, 3] From e98388266e3ee3c8d462a6b088515aa3a6fc40e9 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Thu, 9 Jan 2025 09:53:14 -0600 Subject: [PATCH 70/90] ci: Add missing CI for feast-operator image builds (#4184) add feast-operator builds to CI Signed-off-by: Tommy Hughes --- .github/workflows/build_wheels.yml | 2 +- .github/workflows/master_only.yml | 2 +- Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 39713924111..a538970714f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -93,7 +93,7 @@ jobs: needs: get-version strategy: matrix: - component: [ feature-server, feature-server-java, feature-transformation-server ] + component: [ feature-server, feature-server-java, feature-transformation-server, feast-operator ] env: REGISTRY: feastdev steps: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 446c3b1f3be..a04d767eb5e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - component: [ feature-server-java, feature-transformation-server ] + component: [ feature-server, feature-server-java, feature-transformation-server, feast-operator ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: gcr.io/kf-feast diff --git a/Makefile b/Makefile index 0d08f002bb3..8252f48d464 100644 --- a/Makefile +++ b/Makefile @@ -471,7 +471,7 @@ kill-trino-locally: # Docker -build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-feature-server-docker build-feature-transformation-server-docker build-feature-server-java-docker build-feast-operator-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) From 1d47cb6d9218bf39a1095a48fc773d781c787fd9 Mon Sep 17 00:00:00 2001 From: Dani Date: Thu, 9 Jan 2025 17:19:24 +0100 Subject: [PATCH 71/90] docs: Add missing unpacking operator in Permission objects docs (#4884) docs: add missing unpacking operator in Permission objects docs Signed-off-by: boliri --- docs/getting-started/concepts/permission.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/concepts/permission.md b/docs/getting-started/concepts/permission.md index a6353579687..8db67032878 100644 --- a/docs/getting-started/concepts/permission.md +++ b/docs/getting-started/concepts/permission.md @@ -69,7 +69,7 @@ Permission( name="feature-reader", types=[FeatureView, FeatureService], policy=RoleBasedPolicy(roles=["super-reader"]), - actions=[AuthzedAction.DESCRIBE, READ], + actions=[AuthzedAction.DESCRIBE, *READ], ) ``` From 4b190247c94bb8f863c0ddb959457bde4c371ecb Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 10 Jan 2025 09:11:14 -0500 Subject: [PATCH 72/90] chore: Updating Milvus Online Store implementation to use Milvus lite for local mode (#4911) * chore: Updating workflow to use custom version for get highest semver step Signed-off-by: Francisco Javier Arceo * removing print statements Signed-off-by: Francisco Javier Arceo * removing feature_store.yaml and test_workflow.py Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- .../milvus_online_store/milvus.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index 8d5405c428a..f2283387a0e 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -108,12 +108,16 @@ class MilvusOnlineStore(OnlineStore): def _connect(self, config: RepoConfig) -> MilvusClient: if not self.client: - self.client = MilvusClient( - url=f"{config.online_store.host}:{config.online_store.port}", - token=f"{config.online_store.username}:{config.online_store.password}" - if config.online_store.username and config.online_store.password - else "", - ) + if config.provider == "local": + print("Connecting to Milvus in local mode using ./milvus_demo.db") + self.client = MilvusClient("./milvus_demo.db") + else: + self.client = MilvusClient( + url=f"{config.online_store.host}:{config.online_store.port}", + token=f"{config.online_store.username}:{config.online_store.password}" + if config.online_store.username and config.online_store.password + else "", + ) return self.client def _get_collection(self, config: RepoConfig, table: FeatureView) -> Dict[str, Any]: @@ -247,7 +251,8 @@ def online_write_batch( progress(1) self.client.insert( - collection_name=collection["collection_name"], data=entity_batch_to_insert + collection_name=collection["collection_name"], + data=entity_batch_to_insert, ) def online_read( From a3dfbd52cf70a1c17a0bcfab5f841a1ee79aecb6 Mon Sep 17 00:00:00 2001 From: Cansu <31902747+ckavili@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:44:33 +0100 Subject: [PATCH 73/90] =?UTF-8?q?=F0=9F=8D=AD=20ADD=20-=20Route=20definiti?= =?UTF-8?q?on=20(#4914)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cansu Kavili --- infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 3 ++- .../feast-feature-server/templates/route.yaml | 18 ++++++++++++++++++ infra/charts/feast-feature-server/values.yaml | 4 ++++ 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 infra/charts/feast-feature-server/templates/route.yaml diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 15cb6141236..a88a067f9d9 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.42.0 +version: 0.42.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index a1578196b91..1c3e17993ff 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -60,4 +60,5 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | service.port | int | `80` | | | service.type | string | `"ClusterIP"` | | | serviceAccount.name | string | `""` | | -| tolerations | list | `[]` | | \ No newline at end of file +| tolerations | list | `[]` | | +| route.enabled | bool | `false` | | \ No newline at end of file diff --git a/infra/charts/feast-feature-server/templates/route.yaml b/infra/charts/feast-feature-server/templates/route.yaml new file mode 100644 index 00000000000..2f4d36d9e5a --- /dev/null +++ b/infra/charts/feast-feature-server/templates/route.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.route.enabled) (eq .Values.feast_mode "ui") }} +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: {{ include "feast-feature-server.fullname" . }} + labels: + {{- include "feast-feature-server.labels" . | nindent 4 }} +spec: + to: + kind: Service + name: {{ include "feast-feature-server.fullname" . }} + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +{{- end}} \ No newline at end of file diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index ed54d328d10..9a0d2986631 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -74,3 +74,7 @@ livenessProbe: readinessProbe: initialDelaySeconds: 20 periodSeconds: 10 + +# to create OpenShift Route object for UI +route: + enabled: false \ No newline at end of file From a8aeb79830f12358c2355be44fca68e61992cb46 Mon Sep 17 00:00:00 2001 From: Niklas von Maltzahn Date: Fri, 10 Jan 2025 17:22:38 +0200 Subject: [PATCH 74/90] feat: Add date field support to spark (#4913) * add date field support to spark Signed-off-by: niklasvm * add support for lists of dates Signed-off-by: niklasvm --------- Signed-off-by: niklasvm --- sdk/python/feast/type_map.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 000e9cdae4e..7e2b8a5362e 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -815,6 +815,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType: "float": ValueType.FLOAT, "boolean": ValueType.BOOL, "timestamp": ValueType.UNIX_TIMESTAMP, + "date": ValueType.UNIX_TIMESTAMP, "array": ValueType.BYTES_LIST, "array": ValueType.STRING_LIST, "array": ValueType.INT32_LIST, @@ -824,6 +825,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType: "array": ValueType.FLOAT_LIST, "array": ValueType.BOOL_LIST, "array": ValueType.UNIX_TIMESTAMP_LIST, + "array": ValueType.UNIX_TIMESTAMP_LIST, } if spark_type_as_str.startswith("decimal"): spark_type_as_str = "decimal" From e5527adb1284bdbbafd2b436f872583220cb956d Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 10 Jan 2025 16:09:02 -0500 Subject: [PATCH 75/90] chore: Updating tests to allow for the CLIRunner to use Milvus, also have to handle special case of not running apply and teardown (#4915) * chore: Updating tests to allow for the CLIRunner to use Milvus, also have to handle special case of not running apply and teardown Signed-off-by: Francisco Javier Arceo * Adding cleanup Signed-off-by: Francisco Javier Arceo * adding example repo Signed-off-by: Francisco Javier Arceo * changing defualt to FLAT for local implementation Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- .../milvus_online_store/milvus.py | 21 +- .../example_repos/example_rag_feature_repo.py | 38 ++++ .../online_store/test_online_retrieval.py | 180 ++++++++++++++++++ sdk/python/tests/utils/cli_repo_creator.py | 87 ++++++--- 4 files changed, 293 insertions(+), 33 deletions(-) create mode 100644 sdk/python/tests/example_repos/example_rag_feature_repo.py diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index f2283387a0e..7e840622a8a 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union from pydantic import StrictStr @@ -84,9 +85,10 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """ type: Literal["milvus"] = "milvus" + path: Optional[StrictStr] = "data/online_store.db" host: Optional[StrictStr] = "localhost" port: Optional[int] = 19530 - index_type: Optional[str] = "IVF_FLAT" + index_type: Optional[str] = "FLAT" metric_type: Optional[str] = "L2" embedding_dim: Optional[int] = 128 vector_enabled: Optional[bool] = True @@ -106,11 +108,24 @@ class MilvusOnlineStore(OnlineStore): client: Optional[MilvusClient] = None _collections: Dict[str, Any] = {} + def _get_db_path(self, config: RepoConfig) -> str: + assert ( + config.online_store.type == "milvus" + or config.online_store.type.endswith("MilvusOnlineStore") + ) + + if config.repo_path and not Path(config.online_store.path).is_absolute(): + db_path = str(config.repo_path / config.online_store.path) + else: + db_path = config.online_store.path + return db_path + def _connect(self, config: RepoConfig) -> MilvusClient: if not self.client: if config.provider == "local": - print("Connecting to Milvus in local mode using ./milvus_demo.db") - self.client = MilvusClient("./milvus_demo.db") + db_path = self._get_db_path(config) + print(f"Connecting to Milvus in local mode using {db_path}") + self.client = MilvusClient(db_path) else: self.client = MilvusClient( url=f"{config.online_store.host}:{config.online_store.port}", diff --git a/sdk/python/tests/example_repos/example_rag_feature_repo.py b/sdk/python/tests/example_repos/example_rag_feature_repo.py new file mode 100644 index 00000000000..2f55095bc69 --- /dev/null +++ b/sdk/python/tests/example_repos/example_rag_feature_repo.py @@ -0,0 +1,38 @@ +from datetime import timedelta + +from feast import Entity, FeatureView, Field, FileSource +from feast.types import Array, Float32, Int64, UnixTimestamp + +# This is for Milvus +# Note that file source paths are not validated, so there doesn't actually need to be any data +# at the paths for these file sources. Since these paths are effectively fake, this example +# feature repo should not be used for historical retrieval. + +rag_documents_source = FileSource( + path="data/embedded_documents.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", +) + +item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], +) + +document_embeddings = FeatureView( + name="embedded_documents", + entities=[item], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="item_id", dtype=Int64), + Field(name="created_timestamp", dtype=UnixTimestamp), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 83184643f35..5f0796f4eed 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,5 +1,6 @@ import os import platform +import random import sqlite3 import sys import time @@ -561,3 +562,182 @@ def test_sqlite_vec_import() -> None: """).fetchall() result = [(rowid, round(distance, 2)) for rowid, distance in result] assert result == [(2, 2.39), (1, 2.39)] + + +def test_local_milvus() -> None: + import random + + from pymilvus import MilvusClient + + random.seed(42) + VECTOR_LENGTH: int = 768 + COLLECTION_NAME: str = "test_demo_collection" + + client = MilvusClient("./milvus_demo.db") + + for collection in client.list_collections(): + client.drop_collection(collection_name=collection) + client.create_collection( + collection_name=COLLECTION_NAME, + dimension=VECTOR_LENGTH, + ) + assert client.list_collections() == [COLLECTION_NAME] + + docs = [ + "Artificial intelligence was founded as an academic discipline in 1956.", + "Alan Turing was the first person to conduct substantial research in AI.", + "Born in Maida Vale, London, Turing was raised in southern England.", + ] + # Use fake representation with random vectors (vector_length dimension). + vectors = [[random.uniform(-1, 1) for _ in range(VECTOR_LENGTH)] for _ in docs] + data = [ + {"id": i, "vector": vectors[i], "text": docs[i], "subject": "history"} + for i in range(len(vectors)) + ] + + print("Data has", len(data), "entities, each with fields: ", data[0].keys()) + print("Vector dim:", len(data[0]["vector"])) + + insert_res = client.insert(collection_name=COLLECTION_NAME, data=data) + assert insert_res == {"insert_count": 3, "ids": [0, 1, 2], "cost": 0} + + query_vectors = [[random.uniform(-1, 1) for _ in range(VECTOR_LENGTH)]] + + search_res = client.search( + collection_name=COLLECTION_NAME, # target collection + data=query_vectors, # query vectors + limit=2, # number of returned entities + output_fields=["text", "subject"], # specifies fields to be returned + ) + assert [j["id"] for j in search_res[0]] == [0, 1] + query_result = client.query( + collection_name=COLLECTION_NAME, + filter="id == 0", + ) + assert list(query_result[0].keys()) == ["id", "text", "subject", "vector"] + + client.drop_collection(collection_name=COLLECTION_NAME) + + +def test_milvus_lite_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ + + random.seed(42) + n = 10 # number of samples - note: we'll actually double it + vector_length = 10 + runner = CliRunner() + with runner.local_repo( + example_repo_py=get_example_repo("example_rag_feature_repo.py"), + offline_store="file", + online_store="milvus", + apply=False, + teardown=False, + ) as store: + from datetime import timedelta + + from feast import Entity, FeatureView, Field, FileSource + from feast.types import Array, Float32, Int64, UnixTimestamp + + # This is for Milvus + # Note that file source paths are not validated, so there doesn't actually need to be any data + # at the paths for these file sources. Since these paths are effectively fake, this example + # feature repo should not be used for historical retrieval. + + rag_documents_source = FileSource( + path="data/embedded_documents.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", + ) + + item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], + ) + + document_embeddings = FeatureView( + name="embedded_documents", + entities=[item], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="item_id", dtype=Int64), + Field(name="created_timestamp", dtype=UnixTimestamp), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), + ) + + store.apply([rag_documents_source, item, document_embeddings]) + + # Write some data to two tables + document_embeddings_fv = store.get_feature_view(name="embedded_documents") + + provider = store._get_provider() + + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for item_key in item_keys: + data.append( + ( + item_key, + { + "vector": ValueProto( + float_list_val=FloatListProto( + val=np.random.random( + vector_length, + ) + ) + ) + }, + _utc_now(), + _utc_now(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + documents_df = pd.DataFrame( + { + "item_id": [str(i) for i in range(n)], + "vector": [ + np.random.random( + vector_length, + ) + for i in range(n) + ], + "event_timestamp": [_utc_now() for _ in range(n)], + "created_timestamp": [_utc_now() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="embedded_documents", + df=documents_df, + ) + + query_embedding = np.random.random( + vector_length, + ) + result = store.retrieve_online_documents( + feature="embedded_documents:vector", query=query_embedding, top_k=3 + ).to_dict() + + assert "vector" in result + assert "distance" in result + assert len(result["distance"]) == 3 diff --git a/sdk/python/tests/utils/cli_repo_creator.py b/sdk/python/tests/utils/cli_repo_creator.py index e00104081a2..8bb696f7d4e 100644 --- a/sdk/python/tests/utils/cli_repo_creator.py +++ b/sdk/python/tests/utils/cli_repo_creator.py @@ -51,7 +51,14 @@ def run_with_output(self, args: List[str], cwd: Path) -> Tuple[int, bytes]: return e.returncode, e.output @contextmanager - def local_repo(self, example_repo_py: str, offline_store: str): + def local_repo( + self, + example_repo_py: str, + offline_store: str, + online_store: str = "sqlite", + apply=True, + teardown=True, + ): """ Convenience method to set up all the boilerplate for a local feature repo. """ @@ -67,41 +74,61 @@ def local_repo(self, example_repo_py: str, offline_store: str): data_path = Path(data_dir_name) repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( + if online_store == "sqlite": + yaml_config = dedent( f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: local - online_store: - path: {data_path / "online_store.db"} - offline_store: - type: {offline_store} - entity_key_serialization_version: 2 - """ + project: {project_id} + registry: {data_path / "registry.db"} + provider: local + online_store: + path: {data_path / "online_store.db"} + offline_store: + type: {offline_store} + entity_key_serialization_version: 2 + """ ) - ) + elif online_store == "milvus": + yaml_config = dedent( + f""" + project: {project_id} + registry: {data_path / "registry.db"} + provider: local + online_store: + path: {data_path / "online_store.db"} + type: milvus + vector_enabled: true + embedding_dim: 10 + offline_store: + type: {offline_store} + entity_key_serialization_version: 3 + """ + ) + else: + pass + + repo_config.write_text(yaml_config) repo_example = repo_path / "example.py" repo_example.write_text(example_repo_py) - result = self.run(["apply"], cwd=repo_path) - stdout = result.stdout.decode("utf-8") - stderr = result.stderr.decode("utf-8") - print(f"Apply stdout:\n{stdout}") - print(f"Apply stderr:\n{stderr}") - assert ( - result.returncode == 0 - ), f"stdout: {result.stdout}\nstderr: {result.stderr}" + if apply: + result = self.run(["apply"], cwd=repo_path) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + print(f"Apply stdout:\n{stdout}") + print(f"Apply stderr:\n{stderr}") + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\nstderr: {result.stderr}" yield FeatureStore(repo_path=str(repo_path), config=None) - result = self.run(["teardown"], cwd=repo_path) - stdout = result.stdout.decode("utf-8") - stderr = result.stderr.decode("utf-8") - print(f"Apply stdout:\n{stdout}") - print(f"Apply stderr:\n{stderr}") - assert ( - result.returncode == 0 - ), f"stdout: {result.stdout}\nstderr: {result.stderr}" + if teardown: + result = self.run(["teardown"], cwd=repo_path) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + print(f"Apply stdout:\n{stdout}") + print(f"Apply stderr:\n{stderr}") + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\nstderr: {result.stderr}" From e01e51076f5d8fe5be459037bd254e6f94e0cb0f Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:38:01 -0500 Subject: [PATCH 76/90] feat: Adding EnvFrom support for the OptionalConfigs type to the Go Operator (#4909) * Adding EnvFrom support for the OptionalConfigs type to the feast go operator Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> * * Refactored the code to avoid the redundent code. moved common code to util.go * Incorporated code review comments. Added assertion for environment variables at the container level. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --------- Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../api/v1alpha1/featurestore_types.go | 1 + .../api/v1alpha1/zz_generated.deepcopy.go | 11 + .../crd/bases/feast.dev_featurestores.yaml | 246 ++++++++++++++++++ infra/feast-operator/dist/install.yaml | 246 ++++++++++++++++++ .../featurestore_controller_db_store_test.go | 8 +- .../featurestore_controller_ephemeral_test.go | 14 +- ...restore_controller_kubernetes_auth_test.go | 17 +- ...eaturestore_controller_objectstore_test.go | 7 +- .../featurestore_controller_oidc_auth_test.go | 11 +- .../featurestore_controller_pvc_test.go | 11 +- .../featurestore_controller_test.go | 39 +-- ...featurestore_controller_test_utils_test.go | 147 +++++++++++ .../internal/controller/services/services.go | 3 + 13 files changed, 724 insertions(+), 37 deletions(-) create mode 100644 infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index f8d0c0f7da5..6eeef15d075 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -296,6 +296,7 @@ type DefaultConfigs struct { // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` + EnvFrom *[]corev1.EnvFromSource `json:"envFrom,omitempty"` ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index f1e05030880..72e6fc72007 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -474,6 +474,17 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { } } } + if in.EnvFrom != nil { + in, out := &in.EnvFrom, &out.EnvFrom + *out = new([]v1.EnvFromSource) + if **in != nil { + in, out := *in, *out + *out = make([]v1.EnvFromSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } if in.ImagePullPolicy != nil { in, out := &in.ImagePullPolicy, &out.ImagePullPolicy *out = new(v1.PullPolicy) diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index bd0f5aa61de..270cf4d353c 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -221,6 +221,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -589,6 +630,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -976,6 +1058,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -1485,6 +1608,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -1858,6 +2022,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -2253,6 +2458,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index ef2b1f2272c..af0761a5a6d 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -229,6 +229,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -597,6 +638,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -984,6 +1066,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -1493,6 +1616,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -1866,6 +2030,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: @@ -2261,6 +2466,47 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 0bde0dfd7b9..48013c453c8 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -202,10 +202,12 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(k8sClient.Create(ctx, secret)).To(Succeed()) } + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), @@ -256,6 +258,8 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the secrets") Expect(k8sClient.Delete(ctx, onlineSecret)).To(Succeed()) Expect(k8sClient.Delete(ctx, offlineSecret)).To(Succeed()) @@ -598,6 +602,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) Expect(offlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -615,6 +620,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) Expect(onlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*onlineContainer) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index 70ac81a056d..dbf21a9d918 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -62,11 +62,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { registryPath := "/data/registry.db" BeforeEach(func() { + createEnvFromSecretAndConfigMap() By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ Type: offlineType, @@ -97,6 +98,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -141,6 +144,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(onlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -309,9 +313,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) Expect(offlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) + //check envFrom for offlineContainer + assertEnvFrom(*offlineContainer) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -434,6 +442,10 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) + //check envFrom + // Validate `envFrom` for ConfigMap and Secret + assertEnvFrom(*onlineContainer) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 57a73eb0eb2..c4c40caedc6 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -59,10 +59,12 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { roles := []string{"reader", "writer"} BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} @@ -75,6 +77,8 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) }) @@ -126,6 +130,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -416,9 +421,12 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(repoConfig).To(Equal(&testConfig)) // check offline - env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*offlineContainer) + // check offline config fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) @@ -432,9 +440,12 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(repoConfig).To(Equal(&testConfig)) // check online - env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*onlineContainer) + // check online config fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index aff36f338e7..6b287673c4a 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -64,11 +64,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { } BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil resource.Spec.Services.Registry = &feastdevv1alpha1.Registry{ @@ -81,7 +83,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, }, } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) @@ -90,6 +91,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index 08c92a88a97..913ab2695ed 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -70,10 +70,12 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(k8sClient.Create(ctx, oidcSecret)).To(Succeed()) } + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, @@ -82,6 +84,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } + }) AfterEach(func() { resource := &feastdevv1alpha1.FeatureStore{} @@ -97,6 +100,8 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -148,6 +153,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -222,6 +228,9 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + assertEnvFrom(*services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)) + assertEnvFrom(*services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)) + // check Feast Role feastRole := &rbacv1.Role{} err = k8sClient.Get(ctx, types.NamespacedName{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index 887d9070efb..fa40a34d955 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -75,11 +75,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { registryMountedPath := path.Join(registryMountPath, registryPath) BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ Type: offlineType, @@ -125,6 +127,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -191,6 +195,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.Resources).To(Equal(expectedResources)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -283,6 +288,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) Expect(offlineVolMount.Name).To(Equal(offlinePvcName)) + assertEnvFrom(*offlineContainer) + // check offline pvc pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -307,6 +314,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) + assertEnvFrom(*onlineContainer) + // check online pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 71b5d400f87..b4d5befe4ef 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -402,11 +402,13 @@ var _ = Describe("FeatureStore Controller", func() { featurestore := &feastdevv1alpha1.FeatureStore{} BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) @@ -417,6 +419,9 @@ var _ = Describe("FeatureStore Controller", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + // Delete ConfigMap + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -461,6 +466,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -627,6 +633,8 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*offlineContainer) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -645,6 +653,8 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*onlineContainer) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -1207,33 +1217,6 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { - return &feastdevv1alpha1.FeatureStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: "default", - }, - Spec: feastdevv1alpha1.FeatureStoreSpec{ - FeastProject: feastProject, - Services: &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{}, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, - }, - }, - }, - }, - }, - } -} - func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar { for _, e := range envs { if e.Name == services.TmpFeatureStoreYamlEnvVar { diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go new file mode 100644 index 00000000000..57fa87ce733 --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go @@ -0,0 +1,147 @@ +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func assertEnvFrom(container corev1.Container) { + envFrom := container.EnvFrom + Expect(envFrom).NotTo(BeNil()) + checkEnvFromCounter := 0 + + for _, source := range envFrom { + if source.ConfigMapRef != nil && source.ConfigMapRef.Name == "example-configmap" { + checkEnvFromCounter += 1 + // Simulate retrieval of ConfigMap data and validate + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Name: source.ConfigMapRef.Name, + Namespace: "default", + }, configMap) + Expect(err).NotTo(HaveOccurred()) + // Validate a specific key-value pair from the ConfigMap + Expect(configMap.Data["example-key"]).To(Equal("example-value")) + } + + if source.SecretRef != nil && source.SecretRef.Name == "example-secret" { + checkEnvFromCounter += 1 + // Simulate retrieval of Secret data and validate + secret := &corev1.Secret{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Name: source.SecretRef.Name, + Namespace: "default", + }, secret) + Expect(err).NotTo(HaveOccurred()) + // Validate a specific key-value pair from the Secret + Expect(string(secret.Data["secret-key"])).To(Equal("secret-value")) + } + } + Expect(checkEnvFromCounter).To(Equal(2)) +} + +func createEnvFromSecretAndConfigMap() { + By("creating the config map and secret for envFrom") + envFromConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-configmap", + Namespace: "default", + }, + Data: map[string]string{"example-key": "example-value"}, + } + err := k8sClient.Create(context.TODO(), envFromConfigMap) + Expect(err).ToNot(HaveOccurred()) + + envFromSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-secret", + Namespace: "default", + }, + StringData: map[string]string{"secret-key": "secret-value"}, + } + err = k8sClient.Create(context.TODO(), envFromSecret) + Expect(err).ToNot(HaveOccurred()) +} + +func deleteEnvFromSecretAndConfigMap() { + // Delete ConfigMap + By("Deleting the configmap and secret for envFrom") + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-configmap", + Namespace: "default", + }, + } + err := k8sClient.Delete(context.TODO(), configMap) + Expect(err).ToNot(HaveOccurred()) + + // Delete Secret + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-secret", + Namespace: "default", + }, + } + err = k8sClient.Delete(context.TODO(), secret) + Expect(err).ToNot(HaveOccurred()) +} + +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar, envFromVar *[]corev1.EnvFromSource) *feastdevv1alpha1.FeatureStore { + return &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + OfflineStore: &feastdevv1alpha1.OfflineStore{ + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + EnvFrom: envFromVar, + }, + }, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{ + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + EnvFrom: envFromVar, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, + }, + }, + }, + }, + }, + } +} + +func withEnvFrom() *[]corev1.EnvFromSource { + + return &[]corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "example-configmap"}, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "example-secret"}, + }, + }, + } + +} diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 32cf91d09ba..16f3e663902 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -708,6 +708,9 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs if optionalConfigs.Env != nil { container.Env = envOverride(container.Env, *optionalConfigs.Env) } + if optionalConfigs.EnvFrom != nil { + container.EnvFrom = *optionalConfigs.EnvFrom + } if optionalConfigs.ImagePullPolicy != nil { container.ImagePullPolicy = *optionalConfigs.ImagePullPolicy } From 7fad7e3a6307e0893ead7d176a636045000a2c5e Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Tue, 14 Jan 2025 14:31:41 -0600 Subject: [PATCH 77/90] chore: Change feature-server and operator base images to ubi9 (#4919) change feature-server and operator base images to ubi9 Signed-off-by: Tommy Hughes --- Makefile | 2 +- infra/feast-operator/Dockerfile | 9 ++--- .../feature_servers/multicloud/Dockerfile | 19 +++-------- .../feature_servers/multicloud/Dockerfile.dev | 33 +++++-------------- 4 files changed, 17 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 8252f48d464..8442b558745 100644 --- a/Makefile +++ b/Makefile @@ -527,7 +527,7 @@ build-feast-operator-docker: # Dev images build-feature-server-dev: - docker buildx build --build-arg VERSION=dev \ + docker buildx build \ -t feastdev/feature-server:dev \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile index aca26f92295..ad6ac7eaaaf 100644 --- a/infra/feast-operator/Dockerfile +++ b/infra/feast-operator/Dockerfile @@ -1,9 +1,8 @@ # Build the manager binary -FROM golang:1.21 AS builder +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder ARG TARGETOS ARG TARGETARCH -WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum @@ -23,11 +22,9 @@ COPY internal/controller/ internal/controller/ # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / -COPY --from=builder /workspace/manager . +COPY --from=builder /opt/app-root/src/manager . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index e6afb46aadf..fd3c258d892 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,18 +1,7 @@ -FROM python:3.11-slim-bullseye +FROM registry.access.redhat.com/ubi9/python-311:9.5 -RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus]" - - -RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ - apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && apt update && \ - apt install -y \ - jq \ - libarrow-dev && \ - apt remove -y lsb-release wget && \ - apt-get clean && rm -rf /var/cache/apt/lists +ARG VERSION +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus]"==${VERSION} # modify permissions to support running with a random uid -RUN mkdir -m 775 /.cache -RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 7d31fc3600b..add170fb1dc 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,28 +1,13 @@ -FROM python:3.11-slim-bullseye +FROM registry.access.redhat.com/ubi9/python-311:9.5 -RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir pip-tools +COPY --chown=default . ${APP_ROOT}/src +RUN pip install --no-cache-dir pip-tools && \ + make install-python-ci-dependencies && \ + pip uninstall -y pip-tools -RUN apt update && apt install -y -V ca-certificates lsb-release wget make git curl gcc && \ - curl -sL https://deb.nodesource.com/setup_20.x | bash - && \ - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ - apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && apt update && \ - apt install -y \ - jq \ - nodejs \ - libarrow-dev && \ - npm install -g yarn && \ - apt remove -y lsb-release wget && \ - apt-get clean && rm -rf /var/cache/apt/lists - -COPY . /feast -WORKDIR /feast -RUN make install-python-ci-dependencies && pip cache purge -ENV NPM_TOKEN '//registry.npmjs.org/:_authToken' -RUN make build-ui && yarn cache clean - -WORKDIR / +RUN npm install -S yarn +ENV PATH ${PATH}:${APP_ROOT}/src/node_modules/yarn/bin +RUN make build-ui && yarn cache clean --all # modify permissions to support running with a random uid -RUN mkdir -m 775 /.cache -RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json From 44a48899502bc5d2a3e6be84e507490e2d3e4966 Mon Sep 17 00:00:00 2001 From: Abdul Hameed Date: Tue, 14 Jan 2025 17:32:49 -0500 Subject: [PATCH 78/90] =?UTF-8?q?chore:=20Add=20Kustomize=20structure=20fo?= =?UTF-8?q?r=20Feast=20Operator=20with=20environment-specific=20=E2=80=A6?= =?UTF-8?q?=20(#4921)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Kustomize structure for Feast Operator with environment-specific overlays Signed-off-by: Abdul Hameed --- .../config/overlays/odh/delete-namespace.yaml | 5 +++ .../config/overlays/odh/kustomization.yaml | 35 +++++++++++++++++++ .../config/overlays/odh/params.env | 1 + .../config/overlays/odh/params.yaml | 3 ++ infra/scripts/release/files_to_bump.txt | 1 + 5 files changed, 45 insertions(+) create mode 100644 infra/feast-operator/config/overlays/odh/delete-namespace.yaml create mode 100644 infra/feast-operator/config/overlays/odh/kustomization.yaml create mode 100644 infra/feast-operator/config/overlays/odh/params.env create mode 100644 infra/feast-operator/config/overlays/odh/params.yaml diff --git a/infra/feast-operator/config/overlays/odh/delete-namespace.yaml b/infra/feast-operator/config/overlays/odh/delete-namespace.yaml new file mode 100644 index 00000000000..9a52c0573de --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/delete-namespace.yaml @@ -0,0 +1,5 @@ +$patch: delete +apiVersion: v1 +kind: Namespace +metadata: + name: system diff --git a/infra/feast-operator/config/overlays/odh/kustomization.yaml b/infra/feast-operator/config/overlays/odh/kustomization.yaml new file mode 100644 index 00000000000..b74a0e4a4d7 --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: opendatahub + + +resources: +- ../../default + + +patches: + # patch to remove default `system` namespace in ../../manager/manager.yaml + - path: delete-namespace.yaml + +configMapGenerator: + - name: feast-operator-parameters + envs: + - params.env + +configurations: + - params.yaml + +replacements: + - source: + kind: ConfigMap + name: feast-operator-parameters + version: v1 + fieldPath: data.odh-feast-operator-controller-image + targets: + - select: + kind: Deployment + name: controller-manager + fieldPaths: + - spec.template.spec.containers.[name=manager].image + diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env new file mode 100644 index 00000000000..16c8e1a6fb0 --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -0,0 +1 @@ +odh-feast-operator-controller-image=docker.io/feastdev/feast-operator:0.42.0 diff --git a/infra/feast-operator/config/overlays/odh/params.yaml b/infra/feast-operator/config/overlays/odh/params.yaml new file mode 100644 index 00000000000..43509ff293c --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/params.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/template/spec/containers[]/image + kind: Deployment diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 652bc3cad10..8dabe1104f5 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -14,6 +14,7 @@ infra/feast-helm-operator/Makefile 6 infra/feast-helm-operator/config/manager/kustomization.yaml 8 infra/feast-operator/Makefile 6 infra/feast-operator/config/manager/kustomization.yaml 8 +infra/feast-operator/config/overlays/odh/params.env 1 infra/feast-operator/api/feastversion/version.go 20 java/pom.xml 38 ui/package.json 3 From bd9f071017756e205fbabe6af0d38dfaa9be3d7b Mon Sep 17 00:00:00 2001 From: Niklas von Maltzahn Date: Wed, 15 Jan 2025 03:27:27 +0200 Subject: [PATCH 79/90] feat: Add date support when converting from python to feast types (#4918) add date type to python_type_to_feast_value_type Signed-off-by: niklasvm --- sdk/python/feast/type_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 7e2b8a5362e..d47a4221f60 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -164,6 +164,7 @@ def python_type_to_feast_value_type( "datetime64[ns]": ValueType.UNIX_TIMESTAMP, "datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP, # special dtype of pandas "datetime64[ns, utc]": ValueType.UNIX_TIMESTAMP, + "date": ValueType.UNIX_TIMESTAMP, "category": ValueType.STRING, } From 0db56a2cb5888bc21dbdb331e2b5fc3d33508424 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Tue, 14 Jan 2025 20:38:36 -0500 Subject: [PATCH 80/90] fix: Performance regression in /get-online-features (#4892) * put blocking opps in get online features in threadpool Signed-off-by: Rob Howley * chore: leave comment in path operation Signed-off-by: Rob Howley * use quoted type hint Signed-off-by: Rob Howley * remove hint Signed-off-by: Rob Howley --------- Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 68 +++++++++++++++++------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 1f4918fe7a5..e22ff43b9c2 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -76,6 +76,38 @@ class GetOnlineFeaturesRequest(BaseModel): full_feature_names: bool = False +def _get_features(request: GetOnlineFeaturesRequest, store: "feast.FeatureStore"): + if request.feature_service: + feature_service = store.get_feature_service( + request.feature_service, allow_cache=True + ) + assert_permissions( + resource=feature_service, actions=[AuthzedAction.READ_ONLINE] + ) + features = feature_service # type: ignore + else: + all_feature_views, all_on_demand_feature_views = ( + utils._get_feature_views_to_use( + store.registry, + store.project, + request.features, + allow_cache=True, + hide_dummy_entity=False, + ) + ) + for feature_view in all_feature_views: + assert_permissions( + resource=feature_view, actions=[AuthzedAction.READ_ONLINE] + ) + for od_feature_view in all_on_demand_feature_views: + assert_permissions( + resource=od_feature_view, actions=[AuthzedAction.READ_ONLINE] + ) + features = request.features # type: ignore + + return features + + def get_app( store: "feast.FeatureStore", registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL, @@ -121,33 +153,7 @@ async def lifespan(app: FastAPI): ) async def get_online_features(request: GetOnlineFeaturesRequest) -> Dict[str, Any]: # Initialize parameters for FeatureStore.get_online_features(...) call - if request.feature_service: - feature_service = store.get_feature_service( - request.feature_service, allow_cache=True - ) - assert_permissions( - resource=feature_service, actions=[AuthzedAction.READ_ONLINE] - ) - features = feature_service # type: ignore - else: - all_feature_views, all_on_demand_feature_views = ( - utils._get_feature_views_to_use( - store.registry, - store.project, - request.features, - allow_cache=True, - hide_dummy_entity=False, - ) - ) - for feature_view in all_feature_views: - assert_permissions( - resource=feature_view, actions=[AuthzedAction.READ_ONLINE] - ) - for od_feature_view in all_on_demand_feature_views: - assert_permissions( - resource=od_feature_view, actions=[AuthzedAction.READ_ONLINE] - ) - features = request.features # type: ignore + features = await run_in_threadpool(_get_features, request, store) read_params = dict( features=features, @@ -163,9 +169,13 @@ async def get_online_features(request: GetOnlineFeaturesRequest) -> Dict[str, An ) # Convert the Protobuf object to JSON and return it - return MessageToDict( - response.proto, preserving_proto_field_name=True, float_precision=18 + response_dict = await run_in_threadpool( + MessageToDict, + response.proto, + preserving_proto_field_name=True, + float_precision=18, ) + return response_dict @app.post("/push", dependencies=[Depends(inject_user_details)]) async def push(request: PushFeaturesRequest) -> None: From 40b975b8468de2678af8b191e93495e51af0b6aa Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 05:45:26 -0600 Subject: [PATCH 81/90] fix: Move pre-release image builds to quay.io, retire gcr.io pushes (#4922) move pre-release image builds to quay.io Signed-off-by: Tommy Hughes --- .github/workflows/java_master_only.yml | 2 +- .github/workflows/java_pr.yml | 2 +- .github/workflows/master_only.yml | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index b7f49d14544..16fabbbdeed 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -16,7 +16,7 @@ jobs: component: [feature-server-java] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 3aea4d275e8..40a2a7a7ec9 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -84,7 +84,7 @@ jobs: component: [ feature-server-java ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index a04d767eb5e..55246232855 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -97,7 +97,7 @@ jobs: component: [ feature-server, feature-server-java, feature-transformation-server, feast-operator ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 - name: Set up QEMU @@ -106,11 +106,12 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true - - name: Login to DockerHub + - name: Login to Quay.io uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: quay.io + username: ${{ secrets.QUAYIO_CI_USERNAME }} + password: ${{ secrets.QUAYIO_CI_TOKEN }} - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' with: From 695e49bd93a4c8af2ce5839586295b5e74e1b98e Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 08:34:59 -0600 Subject: [PATCH 82/90] fix: Fix integration build/push for images (#4923) fix integration build/push for images Signed-off-by: Tommy Hughes --- .github/workflows/java_master_only.yml | 6 ++++++ .github/workflows/master_only.yml | 2 +- Makefile | 14 +++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 16fabbbdeed..d38d0d5b5b3 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -40,6 +40,12 @@ jobs: run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAYIO_CI_USERNAME }} + password: ${{ secrets.QUAYIO_CI_TOKEN }} - name: Push image run: make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - name: Push development Docker image diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 55246232855..9d7f66005b9 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - component: [ feature-server, feature-server-java, feature-transformation-server, feast-operator ] + component: [ feature-server-dev, feature-transformation-server, feast-operator ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: quay.io/feastdev-ci diff --git a/Makefile b/Makefile index 8442b558745..c199eb3a5ee 100644 --- a/Makefile +++ b/Makefile @@ -477,11 +477,11 @@ push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) push-feature-server-docker: - docker push $(REGISTRY)/feature-server:$$VERSION + docker push $(REGISTRY)/feature-server:$(VERSION) build-feature-server-docker: - docker buildx build --build-arg VERSION=$$VERSION \ - -t $(REGISTRY)/feature-server:$$VERSION \ + docker buildx build --build-arg VERSION=$(VERSION) \ + -t $(REGISTRY)/feature-server:$(VERSION) \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile --load . push-feature-transformation-server-docker: @@ -531,6 +531,14 @@ build-feature-server-dev: -t feastdev/feature-server:dev \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . +build-feature-server-dev-docker: + docker buildx build \ + -t $(REGISTRY)/feature-server:$(VERSION) \ + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . + +push-feature-server-dev-docker: + docker push $(REGISTRY)/feature-server:$(VERSION) + build-java-docker-dev: make build-java-no-tests REVISION=dev docker buildx build --build-arg VERSION=dev \ From 13c7267b555cca4f3361f34fb384a6fd9f27dedf Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 09:19:05 -0600 Subject: [PATCH 83/90] fix: Fix integration operator push (#4924) fix integration build/push for images Signed-off-by: Tommy Hughes --- infra/feast-operator/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 6984ac66e7f..382ac2cfee0 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -142,7 +142,7 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build -t ${IMG} --load . ## Build feast docker image. .PHONY: feast-ci-dev-docker-img @@ -167,7 +167,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - $(CONTAINER_TOOL) buildx create --name project-v3-builder $(CONTAINER_TOOL) buildx use project-v3-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross --load . - $(CONTAINER_TOOL) buildx rm project-v3-builder rm Dockerfile.cross From 32aaf9aba96c53e1c69577312982472182e99659 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 10:14:13 -0600 Subject: [PATCH 84/90] fix: Remove unnecessary google cloud steps & upgrade docker action versions (#4925) remove unecessary google cloud steps & upgrade docker action versions Signed-off-by: Tommy Hughes --- .github/workflows/java_master_only.yml | 2 +- .github/workflows/master_only.yml | 26 +++++++++----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index d38d0d5b5b3..0307034bdb1 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -41,7 +41,7 @@ jobs: - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAYIO_CI_USERNAME }} diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 9d7f66005b9..439bbe125c2 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -96,33 +96,22 @@ jobs: matrix: component: [ feature-server-dev, feature-transformation-server, feast-operator ] env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: quay.io/feastdev-ci + IMAGE: ${{ matrix.component }} steps: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: install: true - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAYIO_CI_USERNAME }} password: ${{ secrets.QUAYIO_CI_TOKEN }} - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - name: Use gcloud CLI - run: gcloud info - - run: gcloud auth configure-docker --quiet - name: Build image run: | make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} @@ -130,5 +119,8 @@ jobs: run: | make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop - docker push ${REGISTRY}/${{ matrix.component }}:develop + if [[ ${{ matrix.component }} == "feature-server-dev" ]]; then + echo "IMAGE=feature-server" >> $GITHUB_ENV + fi + docker tag ${REGISTRY}/${IMAGE}:${GITHUB_SHA} ${REGISTRY}/${IMAGE}:develop + docker push ${REGISTRY}/${IMAGE}:develop From 02458fd7aad49d5daa5b9836f5abdc4dd81d07bb Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 10:57:13 -0600 Subject: [PATCH 85/90] fix: Change image push to use --all-tags option (#4926) change image push to use --all-tags option Signed-off-by: Tommy Hughes --- .github/workflows/master_only.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 439bbe125c2..840a8007236 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -97,7 +97,6 @@ jobs: component: [ feature-server-dev, feature-transformation-server, feast-operator ] env: REGISTRY: quay.io/feastdev-ci - IMAGE: ${{ matrix.component }} steps: - uses: actions/checkout@v4 - name: Set up QEMU @@ -117,10 +116,10 @@ jobs: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - name: Push image run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - - if [[ ${{ matrix.component }} == "feature-server-dev" ]]; then - echo "IMAGE=feature-server" >> $GITHUB_ENV + if [[ "${{ matrix.component }}" == "feature-server-dev" ]]; then + docker tag ${REGISTRY}/feature-server:${GITHUB_SHA} ${REGISTRY}/feature-server:develop + docker push ${REGISTRY}/feature-server --all-tags + else + docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop + docker push ${REGISTRY}/${{ matrix.component }} --all-tags fi - docker tag ${REGISTRY}/${IMAGE}:${GITHUB_SHA} ${REGISTRY}/${IMAGE}:develop - docker push ${REGISTRY}/${IMAGE}:develop From dd472bad262fed8f3d48ab0012cf6d07b54044cd Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 15 Jan 2025 12:51:07 -0600 Subject: [PATCH 86/90] chore: Upgrade go & operator-sdk versions used for the Operator (#4927) upgrade to operator-sdk 1.38 & go 1.22 Signed-off-by: Tommy Hughes --- .../operator-e2e-integration-tests.yml | 2 +- .github/workflows/operator_pr.yml | 2 +- .github/workflows/release.yml | 2 +- infra/feast-operator/.golangci.yml | 8 +- infra/feast-operator/Dockerfile | 2 +- infra/feast-operator/Makefile | 31 +- infra/feast-operator/README.md | 6 +- .../api/v1alpha1/featurestore_types.go | 12 +- infra/feast-operator/cmd/main.go | 48 ++- .../crd/bases/feast.dev_featurestores.yaml | 324 ++++++++++++--- .../config/default/kustomization.yaml | 12 +- .../default/manager_auth_proxy_patch.yaml | 39 -- .../config/default/manager_metrics_patch.yaml | 4 + .../metrics_service.yaml} | 2 +- .../config/manager/manager.yaml | 1 + .../config/prometheus/monitor.yaml | 11 +- .../config/rbac/auth_proxy_role.yaml | 20 - .../config/rbac/kustomization.yaml | 16 +- .../config/rbac/metrics_auth_role.yaml | 20 + ...ng.yaml => metrics_auth_role_binding.yaml} | 4 +- ...sterrole.yaml => metrics_reader_role.yaml} | 0 infra/feast-operator/dist/install.yaml | 385 ++++++++++++++---- infra/feast-operator/go.mod | 52 ++- infra/feast-operator/go.sum | 112 +++-- .../controller/featurestore_controller.go | 14 +- .../featurestore_controller_ephemeral_test.go | 4 +- .../controller/services/suite_test.go | 4 +- .../internal/controller/services/tls_test.go | 28 +- .../internal/controller/suite_test.go | 4 +- .../test/api/featurestore_types_test.go | 6 +- infra/feast-operator/test/api/suite_test.go | 4 +- .../feast-operator/test/e2e/e2e_suite_test.go | 2 +- infra/feast-operator/test/e2e/e2e_test.go | 16 +- infra/feast-operator/test/utils/utils.go | 6 +- 34 files changed, 860 insertions(+), 343 deletions(-) delete mode 100644 infra/feast-operator/config/default/manager_auth_proxy_patch.yaml create mode 100644 infra/feast-operator/config/default/manager_metrics_patch.yaml rename infra/feast-operator/config/{rbac/auth_proxy_service.yaml => default/metrics_service.yaml} (94%) delete mode 100644 infra/feast-operator/config/rbac/auth_proxy_role.yaml create mode 100644 infra/feast-operator/config/rbac/metrics_auth_role.yaml rename infra/feast-operator/config/rbac/{auth_proxy_role_binding.yaml => metrics_auth_role_binding.yaml} (84%) rename infra/feast-operator/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_reader_role.yaml} (100%) diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index a06e793410e..83b38e52be3 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.21.0' + go-version: 1.22.9 - name: Create KIND cluster run: | diff --git a/.github/workflows/operator_pr.yml b/.github/workflows/operator_pr.yml index 232ccf7d339..2feed8dbf32 100644 --- a/.github/workflows/operator_pr.yml +++ b/.github/workflows/operator_pr.yml @@ -9,7 +9,7 @@ jobs: - name: Install Go uses: actions/setup-go@v5 with: - go-version: 1.21.x + go-version: 1.22.9 - name: Operator tests run: | cd infra/feast-operator/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 79b845b101a..00f65929265 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.21.x + go-version: 1.22.9 - name: Build & version operator-specific release files run: | cd infra/feast-operator/ diff --git a/infra/feast-operator/.golangci.yml b/infra/feast-operator/.golangci.yml index ca69a11f6fd..bf54dc46026 100644 --- a/infra/feast-operator/.golangci.yml +++ b/infra/feast-operator/.golangci.yml @@ -21,7 +21,6 @@ linters: enable: - dupl - errcheck - - exportloopref - goconst - gocyclo - gofmt @@ -32,9 +31,16 @@ linters: - lll - misspell - nakedret + - ginkgolinter - prealloc + - revive - staticcheck - typecheck - unconvert - unparam - unused + +linters-settings: + revive: + rules: + - name: comment-spacings diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile index ad6ac7eaaaf..9fd79d7285a 100644 --- a/infra/feast-operator/Dockerfile +++ b/infra/feast-operator/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +FROM registry.access.redhat.com/ubi9/go-toolset:1.22.9 AS builder ARG TARGETOS ARG TARGETARCH diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 382ac2cfee0..46d4451c989 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -48,11 +48,11 @@ endif # Set the Operator SDK version to use. By default, what is installed on the system is used. # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.37.0 +OPERATOR_SDK_VERSION ?= v1.38.0 # Image URL to use all building/pushing image targets IMG ?= $(IMAGE_TAG_BASE):$(VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.29.0 +ENVTEST_K8S_VERSION = 1.30.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -209,16 +209,16 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) -ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions -KUSTOMIZE_VERSION ?= v5.3.0 -CONTROLLER_TOOLS_VERSION ?= v0.14.0 -ENVTEST_VERSION ?= release-0.17 -GOLANGCI_LINT_VERSION ?= v1.57.2 +KUSTOMIZE_VERSION ?= v5.4.2 +CONTROLLER_TOOLS_VERSION ?= v0.15.0 +ENVTEST_VERSION ?= release-0.18 +GOLANGCI_LINT_VERSION ?= v1.59.1 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -238,20 +238,23 @@ $(ENVTEST): $(LOCALBIN) .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary (ideally with version) # $2 - package url which can be installed # $3 - specific version of package define go-install-tool -@[ -f $(1) ] || { \ +@[ -f "$(1)-$(3)" ] || { \ + echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ set -e; \ package=$(2)@$(3) ;\ echo "Downloading $${package}" ;\ GOBIN=$(LOCALBIN) go install $${package} ;\ -mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ -} +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) endef .PHONY: operator-sdk diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index 3012eb63d4b..9d4b51f37f4 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -4,7 +4,7 @@ This is a K8s Operator that can be used to deploy and manage **Feast**, an open ## Getting Started ### Prerequisites -- go version v1.21.0+ +- go version v1.22 - docker version 17.03+. - kubectl version v1.11.3+. - Access to a Kubernetes v1.11.3+ cluster. @@ -108,8 +108,8 @@ make deploy IMG=/feast-operator: ``` ### Prerequisites -- go version v1.21 -- operator-sdk version v1.37.0 +- go version v1.22 +- operator-sdk version v1.38.0 **NOTE:** Run `make help` for more information on all potential `make` targets diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 6eeef15d075..2028f488285 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -385,11 +385,11 @@ type ServiceHostnames struct { Registry string `json:"registry,omitempty"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status -//+kubebuilder:resource:shortName=feast -//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` -//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=feast +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // FeatureStore is the Schema for the featurestores API type FeatureStore struct { @@ -400,7 +400,7 @@ type FeatureStore struct { Status FeatureStoreStatus `json:"status,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // FeatureStoreList contains a list of FeatureStore type FeatureStoreList struct { diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 23a0309041b..cf7b9a76f21 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -33,13 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "github.com/feast-dev/feast/infra/feast-operator/internal/controller" "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( @@ -51,7 +52,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(feastdevv1alpha1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme } func main() { @@ -60,13 +61,15 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -88,7 +91,6 @@ func main() { c.NextProtos = []string{"http/1.1"} } - tlsOpts := []func(*tls.Config){} if !enableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } @@ -97,13 +99,33 @@ func main() { TLSOpts: tlsOpts, }) + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - }, + Scheme: scheme, + Metrics: metricsServerOptions, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, @@ -142,7 +164,7 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "FeatureStore") os.Exit(1) } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 270cf4d353c..88254d73b9f 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: featurestores.feast.dev spec: group: feast.dev @@ -80,10 +80,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -140,10 +145,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -203,10 +213,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -230,10 +245,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -249,10 +269,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must be @@ -352,10 +377,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -391,10 +421,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -500,10 +535,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -549,10 +589,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -612,10 +657,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -639,10 +689,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -658,10 +713,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must be @@ -763,10 +823,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -807,10 +872,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -923,10 +993,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -976,10 +1051,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1040,10 +1120,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1067,10 +1152,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -1086,10 +1176,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -1191,10 +1286,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1248,10 +1348,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1352,10 +1457,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1402,10 +1512,15 @@ spec: the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1465,10 +1580,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1526,10 +1646,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1590,10 +1715,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1617,10 +1747,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -1636,10 +1771,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -1740,10 +1880,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1780,10 +1925,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1891,10 +2041,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1940,10 +2095,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2004,10 +2164,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2031,10 +2196,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -2050,10 +2220,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -2156,10 +2331,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2202,10 +2382,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2320,10 +2505,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2373,10 +2563,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2440,10 +2635,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2467,10 +2667,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2486,10 +2691,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2595,10 +2805,15 @@ spec: field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2652,10 +2867,15 @@ spec: "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2757,10 +2977,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2807,10 +3032,15 @@ spec: where the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic diff --git a/infra/feast-operator/config/default/kustomization.yaml b/infra/feast-operator/config/default/kustomization.yaml index 957965b9b35..dc1504e24a4 100644 --- a/infra/feast-operator/config/default/kustomization.yaml +++ b/infra/feast-operator/config/default/kustomization.yaml @@ -25,12 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml b/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/infra/feast-operator/config/default/manager_metrics_patch.yaml b/infra/feast-operator/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..2aaef6536f4 --- /dev/null +++ b/infra/feast-operator/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/infra/feast-operator/config/rbac/auth_proxy_service.yaml b/infra/feast-operator/config/default/metrics_service.yaml similarity index 94% rename from infra/feast-operator/config/rbac/auth_proxy_service.yaml rename to infra/feast-operator/config/default/metrics_service.yaml index c2bf4e37939..0207c0469d4 100644 --- a/infra/feast-operator/config/rbac/auth_proxy_service.yaml +++ b/infra/feast-operator/config/default/metrics_service.yaml @@ -12,6 +12,6 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager diff --git a/infra/feast-operator/config/manager/manager.yaml b/infra/feast-operator/config/manager/manager.yaml index 90ef7b48635..c0263e3a1c0 100644 --- a/infra/feast-operator/config/manager/manager.yaml +++ b/infra/feast-operator/config/manager/manager.yaml @@ -62,6 +62,7 @@ spec: - /manager args: - --leader-elect + - --health-probe-bind-address=:8081 image: controller:latest name: manager securityContext: diff --git a/infra/feast-operator/config/prometheus/monitor.yaml b/infra/feast-operator/config/prometheus/monitor.yaml index 55484079677..e76479a1305 100644 --- a/infra/feast-operator/config/prometheus/monitor.yaml +++ b/infra/feast-operator/config/prometheus/monitor.yaml @@ -11,10 +11,19 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key insecureSkipVerify: true selector: matchLabels: diff --git a/infra/feast-operator/config/rbac/auth_proxy_role.yaml b/infra/feast-operator/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 55f87916462..00000000000 --- a/infra/feast-operator/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: feast-operator - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/infra/feast-operator/config/rbac/kustomization.yaml b/infra/feast-operator/config/rbac/kustomization.yaml index 5e4972b5397..d22437a5390 100644 --- a/infra/feast-operator/config/rbac/kustomization.yaml +++ b/infra/feast-operator/config/rbac/kustomization.yaml @@ -9,13 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/infra/feast-operator/config/rbac/metrics_auth_role.yaml b/infra/feast-operator/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..bee99788cf4 --- /dev/null +++ b/infra/feast-operator/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: feast-operator + app.kubernetes.io/managed-by: kustomize + name: metrics-auth-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml b/infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml similarity index 84% rename from infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml rename to infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml index ffa85c82af6..f84b6c4160c 100644 --- a/infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml +++ b/infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml @@ -4,11 +4,11 @@ metadata: labels: app.kubernetes.io/name: feast-operator app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-auth-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/infra/feast-operator/config/rbac/auth_proxy_client_clusterrole.yaml b/infra/feast-operator/config/rbac/metrics_reader_role.yaml similarity index 100% rename from infra/feast-operator/config/rbac/auth_proxy_client_clusterrole.yaml rename to infra/feast-operator/config/rbac/metrics_reader_role.yaml diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index af0761a5a6d..82d6845c229 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -11,7 +11,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: featurestores.feast.dev spec: group: feast.dev @@ -88,10 +88,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -148,10 +153,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -211,10 +221,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -238,10 +253,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -257,10 +277,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must be @@ -360,10 +385,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -399,10 +429,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -508,10 +543,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -557,10 +597,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -620,10 +665,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -647,10 +697,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -666,10 +721,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must be @@ -771,10 +831,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -815,10 +880,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -931,10 +1001,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -984,10 +1059,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1048,10 +1128,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1075,10 +1160,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -1094,10 +1184,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -1199,10 +1294,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1256,10 +1356,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1360,10 +1465,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1410,10 +1520,15 @@ spec: the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1473,10 +1588,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1534,10 +1654,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1598,10 +1723,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1625,10 +1755,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -1644,10 +1779,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -1748,10 +1888,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1788,10 +1933,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1899,10 +2049,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1948,10 +2103,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2012,10 +2172,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2039,10 +2204,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap must @@ -2058,10 +2228,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret must @@ -2164,10 +2339,15 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2210,10 +2390,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2328,10 +2513,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2381,10 +2571,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2448,10 +2643,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2475,10 +2675,15 @@ spec: description: The ConfigMap to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2494,10 +2699,15 @@ spec: description: The Secret to select from properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2603,10 +2813,15 @@ spec: field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2660,10 +2875,15 @@ spec: "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2765,10 +2985,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2815,10 +3040,15 @@ spec: where the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -3118,20 +3348,7 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: feast-operator - name: feast-operator-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: feast-operator - name: feast-operator-proxy-role + name: feast-operator-metrics-auth-role rules: - apiGroups: - authentication.k8s.io @@ -3147,6 +3364,19 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: feast-operator + name: feast-operator-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -3185,11 +3415,11 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: feast-operator - name: feast-operator-proxy-rolebinding + name: feast-operator-metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: feast-operator-proxy-role + name: feast-operator-metrics-auth-role subjects: - kind: ServiceAccount name: feast-operator-controller-manager @@ -3209,7 +3439,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager --- @@ -3236,32 +3466,9 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --leader-elect + - --health-probe-bind-address=:8081 command: - /manager image: feastdev/feast-operator:0.42.0 diff --git a/infra/feast-operator/go.mod b/infra/feast-operator/go.mod index 4e544d819e4..337cb3e80db 100644 --- a/infra/feast-operator/go.mod +++ b/infra/feast-operator/go.mod @@ -1,25 +1,30 @@ module github.com/feast-dev/feast/infra/feast-operator -go 1.21 +go 1.22.9 require ( - github.com/onsi/ginkgo/v2 v2.14.0 - github.com/onsi/gomega v1.30.0 + github.com/onsi/ginkgo/v2 v2.17.1 + github.com/onsi/gomega v1.32.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.2 - k8s.io/apimachinery v0.29.2 - k8s.io/client-go v0.29.2 - sigs.k8s.io/controller-runtime v0.17.3 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 + sigs.k8s.io/controller-runtime v0.18.4 ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -27,12 +32,14 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -47,26 +54,41 @@ require ( github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apiextensions-apiserver v0.29.2 // indirect - k8s.io/component-base v0.29.2 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/infra/feast-operator/go.sum b/infra/feast-operator/go.sum index be475e11018..ee6c96863b6 100644 --- a/infra/feast-operator/go.sum +++ b/infra/feast-operator/go.sum @@ -1,5 +1,11 @@ +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -13,13 +19,17 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -32,15 +42,17 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -51,6 +63,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -78,10 +92,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= -github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -98,10 +112,13 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -110,6 +127,22 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -135,6 +168,8 @@ golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -154,8 +189,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -164,39 +199,50 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= -k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= -k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= -k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= -sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 984bb7c9c26..99be6962b56 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -52,13 +52,13 @@ type FeatureStoreReconciler struct { Scheme *runtime.Scheme } -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update -//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index dbf21a9d918..dba603cbfa6 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -317,7 +317,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - //check envFrom for offlineContainer + // check envFrom for offlineContainer assertEnvFrom(*offlineContainer) fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() @@ -442,7 +442,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - //check envFrom + // check envFrom // Validate `envFrom` for ConfigMap and Secret assertEnvFrom(*onlineContainer) diff --git a/infra/feast-operator/internal/controller/services/suite_test.go b/infra/feast-operator/internal/controller/services/suite_test.go index e1e485f1bf6..5e922bc7e4a 100644 --- a/infra/feast-operator/internal/controller/services/suite_test.go +++ b/infra/feast-operator/internal/controller/services/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 522eb2265b5..5baeb381d7f 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -51,7 +51,7 @@ var _ = Describe("TLS Config", func() { }, } err := feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls := feast.getTlsConfigs(RegistryFeastType) Expect(tls).To(BeNil()) @@ -64,14 +64,14 @@ var _ = Describe("TLS Config", func() { Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err := feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeFalse()) // registry service w/ openshift tls testSetIsOpenShift() feast.Handler.FeatureStore = minimalFeatureStore() err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls = feast.getTlsConfigs(OfflineFeastType) Expect(tls).To(BeNil()) @@ -92,13 +92,13 @@ var _ = Describe("TLS Config", func() { Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // all services w/ openshift tls feast.Handler.FeatureStore = minimalFeatureStoreWithAllServices() err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) repoConfig, err := getClientRepoConfig(feast.Handler.FeatureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) @@ -131,13 +131,13 @@ var _ = Describe("TLS Config", func() { Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeTrue()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // check k8s deployment objects feastDeploy := feast.initFeastDeploy() err = feast.setDeployment(feastDeploy) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) @@ -163,7 +163,7 @@ var _ = Describe("TLS Config", func() { }, } err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls = feast.getTlsConfigs(OfflineFeastType) Expect(tls).To(BeNil()) @@ -184,7 +184,7 @@ var _ = Describe("TLS Config", func() { Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeFalse()) // all services w/ tls and in an openshift cluster @@ -201,7 +201,7 @@ var _ = Describe("TLS Config", func() { }, } err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) repoConfig, err = getClientRepoConfig(feast.Handler.FeatureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) @@ -232,27 +232,27 @@ var _ = Describe("TLS Config", func() { Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // check k8s service objects offlineSvc := feast.initFeastSvc(OfflineFeastType) Expect(offlineSvc.Annotations).To(BeEmpty()) err = feast.setService(offlineSvc, OfflineFeastType) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(offlineSvc.Annotations).NotTo(BeEmpty()) Expect(offlineSvc.Spec.Ports[0].Name).To(Equal(HttpsScheme)) onlineSvc := feast.initFeastSvc(OnlineFeastType) err = feast.setService(onlineSvc, OnlineFeastType) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(onlineSvc.Annotations).To(BeEmpty()) Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) // check k8s deployment objects feastDeploy = feast.initFeastDeploy() err = feast.setDeployment(feastDeploy) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) diff --git a/infra/feast-operator/internal/controller/suite_test.go b/infra/feast-operator/internal/controller/suite_test.go index 38da27cc9c5..51208d6dbb0 100644 --- a/infra/feast-operator/internal/controller/suite_test.go +++ b/infra/feast-operator/internal/controller/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/test/api/featurestore_types_test.go b/infra/feast-operator/test/api/featurestore_types_test.go index 126991266d3..1049436fec9 100644 --- a/infra/feast-operator/test/api/featurestore_types_test.go +++ b/infra/feast-operator/test/api/featurestore_types_test.go @@ -34,7 +34,7 @@ func attemptInvalidCreationAndAsserts(ctx context.Context, featurestore *feastde logger.Info("Creating", "FeatureStore", featurestore) err := k8sClient.Create(ctx, featurestore) logger.Info("Got", "err", err) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring(matcher)) } @@ -350,12 +350,12 @@ func initContext() (context.Context, *feastdevv1alpha1.FeatureStore) { BeforeEach(func() { By("verifying the custom resource FeatureStore is not there") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - Expect(err != nil && errors.IsNotFound(err)) + Expect(err != nil && errors.IsNotFound(err)).To(BeTrue()) }) AfterEach(func() { By("verifying the custom resource FeatureStore is not there") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - Expect(err != nil && errors.IsNotFound(err)) + Expect(err != nil && errors.IsNotFound(err)).To(BeTrue()) }) return ctx, featurestore diff --git a/infra/feast-operator/test/api/suite_test.go b/infra/feast-operator/test/api/suite_test.go index 270742760e7..e8c46a240c1 100644 --- a/infra/feast-operator/test/api/suite_test.go +++ b/infra/feast-operator/test/api/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -74,7 +74,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/test/e2e/e2e_suite_test.go b/infra/feast-operator/test/e2e/e2e_suite_test.go index 8e46d8a5063..c45853a0073 100644 --- a/infra/feast-operator/test/e2e/e2e_suite_test.go +++ b/infra/feast-operator/test/e2e/e2e_suite_test.go @@ -27,6 +27,6 @@ import ( // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting feast-operator suite\n") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting feast-operator suite\n") RunSpecs(t, "e2e suite") } diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index bab57a3c006..e55ac0156a7 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -81,7 +81,7 @@ var _ = Describe("controller", Ordered, func() { By("Validating that the controller-manager deployment is in available state") err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "Deployment %s is not available but expected to be available. \nError: %v\n", controllerDeploymentName, err, )) @@ -89,7 +89,7 @@ var _ = Describe("controller", Ordered, func() { }) AfterAll(func() { - //Add any post clean up code here. + // Add any post clean up code here. By("Uninstalling the feast CRD") cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) _, err := utils.Run(cmd) @@ -158,7 +158,7 @@ var _ = Describe("controller", Ordered, func() { func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", featureStoreName, err)) @@ -176,7 +176,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st for _, deploymentName := range k8sResourceNames { By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "Deployment %s is not available but expected to be available. \nError: %v\n", deploymentName, err, )) @@ -186,7 +186,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st By("Check if the feast client - kubernetes config map exists.") configMapName := feastResourceName + "-client" err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "config map %s is not available but expected to be available. \nError: %v\n", configMapName, err, )) @@ -195,7 +195,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st for _, serviceAccountName := range k8sResourceNames { By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "Service account %s does not exist in namespace %s. Error: %v", serviceAccountName, namespace, err, )) @@ -205,7 +205,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st for _, serviceName := range feastK8sResourceNames { By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "kubernetes service %s is not available but expected to be available. \nError: %v\n", serviceName, err, )) @@ -214,7 +214,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) - Expect(err).To(BeNil(), fmt.Sprintf( + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", featureStoreName, err, )) diff --git a/infra/feast-operator/test/utils/utils.go b/infra/feast-operator/test/utils/utils.go index 1027041273c..9b57f9af61c 100644 --- a/infra/feast-operator/test/utils/utils.go +++ b/infra/feast-operator/test/utils/utils.go @@ -35,7 +35,7 @@ const ( ) func warnError(err error) { - fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } // InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. @@ -52,12 +52,12 @@ func Run(cmd *exec.Cmd) ([]byte, error) { cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) output, err := cmd.CombinedOutput() if err != nil { return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) From 7a1f4dd8b96a40d055467e1e5f72c91167e40484 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Fri, 17 Jan 2025 09:53:23 -0600 Subject: [PATCH 87/90] feat: Operator improvements (#4928) operator improvements Signed-off-by: Tommy Hughes --- .../api/v1alpha1/featurestore_types.go | 21 +- .../api/v1alpha1/zz_generated.deepcopy.go | 26 +- infra/feast-operator/bundle.Dockerfile | 2 +- ...er-manager-metrics-service_v1_service.yaml | 2 +- .../feast-operator.clusterserviceversion.yaml | 68 +- .../manifests/feast.dev_featurestores.yaml | 1960 ++++++++++++++++- .../bundle/metadata/annotations.yaml | 2 +- .../crd/bases/feast.dev_featurestores.yaml | 109 +- .../config/default/kustomization.yaml | 3 + .../default/manager_related_images_patch.yaml | 10 + .../config/manager/manager.yaml | 5 + infra/feast-operator/dist/install.yaml | 114 +- .../featurestore_controller_db_store_test.go | 10 +- .../featurestore_controller_ephemeral_test.go | 12 +- ...restore_controller_kubernetes_auth_test.go | 6 +- .../featurestore_controller_loglevel_test.go | 12 +- ...eaturestore_controller_objectstore_test.go | 14 +- .../featurestore_controller_oidc_auth_test.go | 16 +- .../featurestore_controller_pvc_test.go | 44 +- .../featurestore_controller_test.go | 37 +- .../featurestore_controller_tls_test.go | 10 +- .../controller/services/repo_config_test.go | 50 +- .../internal/controller/services/services.go | 109 +- .../controller/services/services_types.go | 1 + .../internal/controller/services/tls.go | 2 +- .../internal/controller/services/tls_test.go | 14 +- .../internal/controller/services/util.go | 58 +- infra/scripts/release/files_to_bump.txt | 1 + 28 files changed, 2491 insertions(+), 227 deletions(-) create mode 100644 infra/feast-operator/config/default/manager_related_images_patch.yaml diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 2028f488285..b44f776dab4 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -67,9 +68,12 @@ type FeatureStoreSpec struct { // FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. type FeatureStoreServices struct { - OfflineStore *OfflineStore `json:"offlineStore,omitempty"` - OnlineStore *OnlineStore `json:"onlineStore,omitempty"` - Registry *Registry `json:"registry,omitempty"` + OfflineStore *OfflineStore `json:"offlineStore,omitempty"` + OnlineStore *OnlineStore `json:"onlineStore,omitempty"` + Registry *Registry `json:"registry,omitempty"` + DeploymentStrategy *appsv1.DeploymentStrategy `json:"deploymentStrategy,omitempty"` + // Disable the 'feast repo initialization' initContainer + DisableInitContainers bool `json:"disableInitContainers,omitempty"` } // OfflineStore configures the deployed offline store service @@ -370,12 +374,11 @@ type FeatureStoreStatus struct { // Shows the currently applied feast configuration, including any pertinent defaults Applied FeatureStoreSpec `json:"applied,omitempty"` // ConfigMap in this namespace containing a client `feature_store.yaml` for this feast deployment - ClientConfigMap string `json:"clientConfigMap,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - // Version of feast that's currently deployed - FeastVersion string `json:"feastVersion,omitempty"` - Phase string `json:"phase,omitempty"` - ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` + ClientConfigMap string `json:"clientConfigMap,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + FeastVersion string `json:"feastVersion,omitempty"` + Phase string `json:"phase,omitempty"` + ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` } // ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 72e6fc72007..6fbd44deb70 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,8 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/core/v1" + "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -163,6 +164,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { *out = new(Registry) (*in).DeepCopyInto(*out) } + if in.DeploymentStrategy != nil { + in, out := &in.DeploymentStrategy, &out.DeploymentStrategy + *out = new(v1.DeploymentStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreServices. @@ -465,10 +471,10 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { *out = *in if in.Env != nil { in, out := &in.Env, &out.Env - *out = new([]v1.EnvVar) + *out = new([]corev1.EnvVar) if **in != nil { in, out := *in, *out - *out = make([]v1.EnvVar, len(*in)) + *out = make([]corev1.EnvVar, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -476,10 +482,10 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { } if in.EnvFrom != nil { in, out := &in.EnvFrom, &out.EnvFrom - *out = new([]v1.EnvFromSource) + *out = new([]corev1.EnvFromSource) if **in != nil { in, out := *in, *out - *out = make([]v1.EnvFromSource, len(*in)) + *out = make([]corev1.EnvFromSource, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -487,12 +493,12 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { } if in.ImagePullPolicy != nil { in, out := &in.ImagePullPolicy, &out.ImagePullPolicy - *out = new(v1.PullPolicy) + *out = new(corev1.PullPolicy) **out = **in } if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } } @@ -512,7 +518,7 @@ func (in *PvcConfig) DeepCopyInto(out *PvcConfig) { *out = *in if in.Ref != nil { in, out := &in.Ref, &out.Ref - *out = new(v1.LocalObjectReference) + *out = new(corev1.LocalObjectReference) **out = **in } if in.Create != nil { @@ -537,7 +543,7 @@ func (in *PvcCreate) DeepCopyInto(out *PvcCreate) { *out = *in if in.AccessModes != nil { in, out := &in.AccessModes, &out.AccessModes - *out = make([]v1.PersistentVolumeAccessMode, len(*in)) + *out = make([]corev1.PersistentVolumeAccessMode, len(*in)) copy(*out, *in) } if in.StorageClassName != nil { @@ -737,7 +743,7 @@ func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.LocalObjectReference) + *out = new(corev1.LocalObjectReference) **out = **in } out.SecretKeyNames = in.SecretKeyNames diff --git a/infra/feast-operator/bundle.Dockerfile b/infra/feast-operator/bundle.Dockerfile index ab3f14a9da4..685b137b92a 100644 --- a/infra/feast-operator/bundle.Dockerfile +++ b/infra/feast-operator/bundle.Dockerfile @@ -6,7 +6,7 @@ LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ LABEL operators.operatorframework.io.bundle.package.v1=feast-operator LABEL operators.operatorframework.io.bundle.channels.v1=alpha -LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.37.0 +LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.38.0 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4 diff --git a/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml b/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml index e0cd9dc2545..913517e198a 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml @@ -12,7 +12,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager status: diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 245db443581..bfd32b5a830 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -16,10 +16,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-11-01T13:05:11Z" - operators.operatorframework.io/builder: operator-sdk-v1.37.0 + createdAt: "2025-01-16T22:15:56Z" + operators.operatorframework.io/builder: operator-sdk-v1.38.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.41.0 + name: feast-operator.v0.42.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -54,6 +54,8 @@ spec: - "" resources: - configmaps + - persistentvolumeclaims + - serviceaccounts - services verbs: - create @@ -62,6 +64,13 @@ spec: - list - update - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list - apiGroups: - feast.dev resources: @@ -88,6 +97,18 @@ spec: - get - patch - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - authentication.k8s.io resources: @@ -122,35 +143,17 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --leader-elect + - --health-probe-bind-address=:8081 command: - /manager - image: feastdev/feast-operator:0.41.0 + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.42.0 + - name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine + image: feastdev/feast-operator:0.42.0 livenessProbe: httpGet: path: /healthz @@ -239,4 +242,9 @@ spec: provider: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ - version: 0.41.0 + relatedImages: + - image: docker.io/feastdev/feature-server:0.42.0 + name: feature-server + - image: docker.io/fullstorydev/grpcurl:v1.9.1-alpine + name: grpc-curl + version: 0.42.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 2142e093eb1..ff1a77936f2 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 creationTimestamp: null name: featurestores.feast.dev spec: @@ -48,6 +48,57 @@ spec: spec: description: FeatureStoreSpec defines the desired state of FeatureStore properties: + authz: + description: AuthzConfig defines the authorization settings for the + deployed Feast services. + properties: + kubernetes: + description: |- + KubernetesAuthz provides a way to define the authorization settings using Kubernetes RBAC resources. + https://kubernetes.io/docs/reference/access-authn-authz/rbac/ + properties: + roles: + description: |- + The Kubernetes RBAC roles to be deployed in the same namespace of the FeatureStore. + Roles are managed by the operator and created with an empty list of rules. + See the Feast permission model at https://docs.feast.dev/getting-started/concepts/permission + The feature store admin is not obligated to manage roles using the Feast operator, roles can be managed independently. + This configuration option is only providing a way to automate this procedure. + Important note: the operator cannot ensure that these roles will match the ones used in the configured Feast permissions. + items: + type: string + type: array + type: object + oidc: + description: |- + OidcAuthz defines the authorization settings for deployments using an Open ID Connect identity provider. + https://auth0.com/docs/authenticate/protocols/openid-connect-protocol + properties: + secretRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + type: object + x-kubernetes-validations: + - message: One selection required between kubernetes or oidc. + rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, c)' feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an @@ -55,9 +106,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -94,10 +199,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -157,10 +267,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -175,12 +290,223 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OfflineStorePersistence configures the persistence + settings for the offline store service + properties: + file: + description: OfflineStoreFilePersistence configures the + file-based persistence for the offline store service + properties: + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref and + create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and must + not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: + enum: + - file + - dask + - duckdb + type: string + type: object + store: + description: OfflineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the secret + key. "registry_type" & "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.offline + - bigquery + - redshift + - spark + - postgres + - trino + - redis + - athena + - mssql + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -237,6 +563,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -274,10 +643,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -337,10 +711,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -355,12 +734,237 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OnlineStorePersistence configures the persistence + settings for the online store service + properties: + file: + description: OnlineStoreFilePersistence configures the + file-based persistence for the offline store service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref and + create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and must + not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: object + x-kubernetes-validations: + - message: Ephemeral stores must have absolute paths. + rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') + : true' + - message: PVC path must be a file name only, with no + slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: Online store does not support S3 or GS buckets. + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + store: + description: OnlineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the secret + key. "registry_type" & "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.online + - redis + - ikv + - datastore + - dynamodb + - bigtable + - postgres + - cassandra + - mysql + - hazelcast + - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -417,6 +1021,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One selection @@ -458,10 +1105,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -522,10 +1174,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -540,12 +1197,237 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: RegistryPersistence configures the persistence + settings for the registry service + properties: + file: + description: RegistryFilePersistence configures the + file-based persistence for the registry service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + s3_additional_kwargs: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-validations: + - message: Registry files must use absolute paths + or be S3 ('s3://') or GS ('gs://') object store + URIs. + rule: '(!has(self.pvc) && has(self.path)) ? (self.path.startsWith(''/'') + || self.path.startsWith(''s3://'') || self.path.startsWith(''gs://'')) + : true' + - message: PVC path must be a file name only, with + no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: PVC persistence does not support S3 or + GS object store URIs. + rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: Additional S3 settings are available only + for S3 object store URIs. + rule: '(has(self.s3_additional_kwargs) && has(self.path)) + ? self.path.startsWith(''s3://'') : true' + store: + description: RegistryDBStorePersistence configures + the DB store persistence for the registry service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - sql + - snowflake.registry + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -603,6 +1485,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -626,6 +1551,37 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift cluster, + this is configured by default when the remote feast + registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for the + client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap where + the TLS cert resides + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -646,6 +1602,58 @@ spec: description: Shows the currently applied feast configuration, including any pertinent defaults properties: + authz: + description: AuthzConfig defines the authorization settings for + the deployed Feast services. + properties: + kubernetes: + description: |- + KubernetesAuthz provides a way to define the authorization settings using Kubernetes RBAC resources. + https://kubernetes.io/docs/reference/access-authn-authz/rbac/ + properties: + roles: + description: |- + The Kubernetes RBAC roles to be deployed in the same namespace of the FeatureStore. + Roles are managed by the operator and created with an empty list of rules. + See the Feast permission model at https://docs.feast.dev/getting-started/concepts/permission + The feature store admin is not obligated to manage roles using the Feast operator, roles can be managed independently. + This configuration option is only providing a way to automate this procedure. + Important note: the operator cannot ensure that these roles will match the ones used in the configured Feast permissions. + items: + type: string + type: array + type: object + oidc: + description: |- + OidcAuthz defines the authorization settings for deployments using an Open ID Connect identity provider. + https://auth0.com/docs/authenticate/protocols/openid-connect-protocol + properties: + secretRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + type: object + x-kubernetes-validations: + - message: One selection required between kubernetes or oidc. + rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, + c)' feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start @@ -653,9 +1661,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -692,10 +1754,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -756,10 +1823,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -774,12 +1846,226 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OfflineStorePersistence configures the persistence + settings for the offline store service + properties: + file: + description: OfflineStoreFilePersistence configures + the file-based persistence for the offline store + service + properties: + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: + enum: + - file + - dask + - duckdb + type: string + type: object + store: + description: OfflineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.offline + - bigquery + - redshift + - spark + - postgres + - trino + - redis + - athena + - mssql + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -837,6 +2123,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -874,10 +2203,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -938,10 +2272,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -956,12 +2295,241 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OnlineStorePersistence configures the persistence + settings for the online store service + properties: + file: + description: OnlineStoreFilePersistence configures + the file-based persistence for the offline store + service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: object + x-kubernetes-validations: + - message: Ephemeral stores must have absolute paths. + rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') + : true' + - message: PVC path must be a file name only, with + no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: Online store does not support S3 or GS + buckets. + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + store: + description: OnlineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.online + - redis + - ikv + - datastore + - dynamodb + - bigtable + - postgres + - cassandra + - mysql + - hazelcast + - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -1019,6 +2587,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One @@ -1060,10 +2671,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1127,10 +2743,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1145,12 +2766,242 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: RegistryPersistence configures the persistence + settings for the registry service + properties: + file: + description: RegistryFilePersistence configures + the file-based persistence for the registry + service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new + PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing + field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between + ref and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' + and must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + s3_additional_kwargs: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-validations: + - message: Registry files must use absolute paths + or be S3 ('s3://') or GS ('gs://') object + store URIs. + rule: '(!has(self.pvc) && has(self.path)) ? + (self.path.startsWith(''/'') || self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: PVC path must be a file name only, + with no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: PVC persistence does not support S3 + or GS object store URIs. + rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: Additional S3 settings are available + only for S3 object store URIs. + rule: '(has(self.s3_additional_kwargs) && has(self.path)) + ? self.path.startsWith(''s3://'') : true' + store: + description: RegistryDBStorePersistence configures + the DB store persistence for the registry service + properties: + secretKeyName: + description: By default, the selected store + "type" is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should + be placed as-is from the "feature_store.yaml" + under the secret key. "registry_type" & + "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - sql + - snowflake.registry + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or + store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -1208,6 +3059,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for + a feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, + where TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -1231,6 +3125,37 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift + cluster, this is configured by default when the + remote feast registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for + the client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap + where the TLS cert resides + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -1319,7 +3244,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string diff --git a/infra/feast-operator/bundle/metadata/annotations.yaml b/infra/feast-operator/bundle/metadata/annotations.yaml index bf929b9755b..5e280a43e24 100644 --- a/infra/feast-operator/bundle/metadata/annotations.yaml +++ b/infra/feast-operator/bundle/metadata/annotations.yaml @@ -5,7 +5,7 @@ annotations: operators.operatorframework.io.bundle.metadata.v1: metadata/ operators.operatorframework.io.bundle.package.v1: feast-operator operators.operatorframework.io.bundle.channels.v1: alpha - operators.operatorframework.io.metrics.builder: operator-sdk-v1.37.0 + operators.operatorframework.io.metrics.builder: operator-sdk-v1.38.0 operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 operators.operatorframework.io.metrics.project_layout: go.kubebuilder.io/v4 diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 88254d73b9f..a509caf329f 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -109,6 +109,60 @@ spec: description: FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -1610,6 +1664,60 @@ spec: description: FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -3136,7 +3244,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string diff --git a/infra/feast-operator/config/default/kustomization.yaml b/infra/feast-operator/config/default/kustomization.yaml index dc1504e24a4..01534ae5cc8 100644 --- a/infra/feast-operator/config/default/kustomization.yaml +++ b/infra/feast-operator/config/default/kustomization.yaml @@ -30,6 +30,9 @@ resources: # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: +- path: manager_related_images_patch.yaml + target: + kind: Deployment # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics - path: manager_metrics_patch.yaml diff --git a/infra/feast-operator/config/default/manager_related_images_patch.yaml b/infra/feast-operator/config/default/manager_related_images_patch.yaml new file mode 100644 index 00000000000..7ad1ab5970f --- /dev/null +++ b/infra/feast-operator/config/default/manager_related_images_patch.yaml @@ -0,0 +1,10 @@ +- op: replace + path: "/spec/template/spec/containers/0/env/0" + value: + name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.42.0 +- op: replace + path: "/spec/template/spec/containers/0/env/1" + value: + name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine diff --git a/infra/feast-operator/config/manager/manager.yaml b/infra/feast-operator/config/manager/manager.yaml index c0263e3a1c0..4259cf8a7e0 100644 --- a/infra/feast-operator/config/manager/manager.yaml +++ b/infra/feast-operator/config/manager/manager.yaml @@ -70,6 +70,11 @@ spec: capabilities: drop: - "ALL" + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: feast:latest + - name: RELATED_IMAGE_GRPC_CURL + value: grpc:latest livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 82d6845c229..ae9a37d8c9c 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -117,6 +117,60 @@ spec: description: FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -1618,6 +1672,60 @@ spec: description: FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -3144,7 +3252,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string @@ -3471,6 +3578,11 @@ spec: - --health-probe-bind-address=:8081 command: - /manager + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.42.0 + - name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine image: feastdev/feast-operator:0.42.0 livenessProbe: httpGet: diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 48013c453c8..1c9fa4bc1a5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -562,7 +562,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -600,7 +600,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { } Expect(repoConfig).To(Equal(testConfig)) - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.Env).To(HaveLen(1)) assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) @@ -617,7 +617,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err).NotTo(HaveOccurred()) Expect(repoConfigOffline).To(Equal(testConfig)) - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) Expect(onlineContainer.Env).To(HaveLen(1)) assertEnvFrom(*onlineContainer) @@ -635,7 +635,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) Expect(repoConfigOnline).To(Equal(testConfig)) - onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(1)) // check client config @@ -698,7 +698,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOnlineContainer(*deploy) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index dba603cbfa6..381a2d40a1d 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -278,7 +278,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -311,7 +311,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { } Expect(repoConfig).To(Equal(testConfig)) - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.Env).To(HaveLen(1)) assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) @@ -331,7 +331,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) Expect(repoConfigOffline).To(Equal(testConfig)) - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(3)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) @@ -405,7 +405,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer = services.GetRegistryContainer(*deploy) env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() @@ -422,7 +422,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfig).To(Equal(testConfig)) // check offline config - offlineContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer = services.GetRegistryContainer(*deploy) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -438,7 +438,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOnlineContainer(*deploy) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index c4c40caedc6..e3a0e8fa646 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -397,7 +397,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check registry config @@ -421,7 +421,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(repoConfig).To(Equal(&testConfig)) // check offline - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -440,7 +440,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(repoConfig).To(Equal(&testConfig)) // check online - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go index 5139e14dd38..f238ca9ee4c 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -165,15 +165,15 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command + command := services.GetRegistryContainer(*deploy).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("ERROR")) - command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command + command = services.GetOfflineContainer(*deploy).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("INFO")) - command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command + command = services.GetOnlineContainer(*deploy).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("DEBUG")) }) @@ -221,13 +221,13 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command + command := services.GetRegistryContainer(*deploy).Command Expect(command).NotTo(ContainElement("--log-level")) - command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command + command = services.GetOfflineContainer(*deploy).Command Expect(command).NotTo(ContainElement("--log-level")) - command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command + command = services.GetOnlineContainer(*deploy).Command Expect(command).NotTo(ContainElement("--log-level")) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index 6b287673c4a..6e9adbf8eb5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -190,9 +190,9 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) - Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) - Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetRegistryContainer(*deploy)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(*deploy)).To(BeNil()) + Expect(services.GetOfflineContainer(*deploy)).To(BeNil()) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) @@ -226,7 +226,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.VolumeMounts).To(HaveLen(1)) }) @@ -285,9 +285,9 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) - Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) - Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetRegistryContainer(*deploy)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(*deploy)).To(BeNil()) + Expect(services.GetOfflineContainer(*deploy)).To(BeNil()) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index 913ab2695ed..23a4a32702b 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -224,12 +224,12 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) - Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) - Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOfflineContainer(*deploy).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(HaveLen(1)) + Expect(services.GetRegistryContainer(*deploy).VolumeMounts).To(HaveLen(1)) - assertEnvFrom(*services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)) - assertEnvFrom(*services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)) + assertEnvFrom(*services.GetOnlineContainer(*deploy)) + assertEnvFrom(*services.GetOfflineContainer(*deploy)) // check Feast Role feastRole := &rbacv1.Role{} @@ -338,7 +338,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check registry config @@ -371,7 +371,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(repoConfig).To(Equal(testConfig)) // check offline - env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check offline config @@ -387,7 +387,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(repoConfig).To(Equal(testConfig)) // check online - env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check online config diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index fa40a34d955..c21f702d559 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -264,6 +264,18 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + ephemeralName := "feast-data" + ephemeralVolume := corev1.Volume{ + Name: ephemeralName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + ephemeralVolMount := corev1.VolumeMount{ + Name: ephemeralName, + MountPath: "/" + ephemeralName, + } + // check deployment deploy := &appsv1.Deployment{} objMeta := feast.GetObjectMeta() @@ -276,13 +288,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).NotTo(ContainElement(ephemeralVolume)) name := feast.GetFeastServiceName(services.RegistryFeastType) regVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) Expect(regVol.Name).To(Equal(name)) Expect(regVol.PersistentVolumeClaim.ClaimName).To(Equal(name)) - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.VolumeMounts).To(HaveLen(3)) + Expect(offlineContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) offlineVolMount := services.GetOfflineVolumeMount(feast.Handler.FeatureStore, offlineContainer.VolumeMounts) Expect(offlineVolMount.MountPath).To(Equal(offlineStoreMountPath)) offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) @@ -308,8 +322,9 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { onlineVol := services.GetOnlineVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) Expect(onlineVol.Name).To(Equal(onlinePvcName)) Expect(onlineVol.PersistentVolumeClaim.ClaimName).To(Equal(onlinePvcName)) - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.VolumeMounts).To(HaveLen(3)) + Expect(onlineContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) onlineVolMount := services.GetOnlineVolumeMount(feast.Handler.FeatureStore, onlineContainer.VolumeMounts) Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) @@ -334,8 +349,9 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { registryVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) Expect(registryVol.Name).To(Equal(registryPvcName)) Expect(registryVol.PersistentVolumeClaim.ClaimName).To(Equal(registryPvcName)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.VolumeMounts).To(HaveLen(3)) + Expect(registryContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) registryVolMount := services.GetRegistryVolumeMount(feast.Handler.FeatureStore, registryContainer.VolumeMounts) Expect(registryVolMount.MountPath).To(Equal(registryMountPath)) Expect(registryVolMount.Name).To(Equal(registryPvcName)) @@ -371,15 +387,19 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig).To(BeNil()) - // check online deployment + // check online deployment/container deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ Name: objMeta.Name, Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(ContainElement(ephemeralVolume)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(HaveLen(3)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) + Expect(services.GetRegistryContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) + Expect(services.GetOfflineContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) // check online pvc is deleted log.FromContext(feast.Handler.Context).Info("Checking deletion of", "PersistentVolumeClaim", deploy.Name) @@ -449,7 +469,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -483,7 +503,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { } Expect(repoConfig).To(Equal(testConfig)) - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -501,7 +521,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(3)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) @@ -582,7 +602,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer = services.GetRegistryContainer(*deploy) env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() @@ -599,7 +619,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfig).To(Equal(testConfig)) // check offline config - offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer = services.GetOfflineContainer(*deploy) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -615,7 +635,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - onlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOfflineContainer(*deploy) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index b4d5befe4ef..9f146bf1f61 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -228,6 +228,7 @@ var _ = Describe("FeatureStore Controller", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) @@ -267,6 +268,11 @@ var _ = Describe("FeatureStore Controller", func() { // change feast project and reconcile resourceNew := resource.DeepCopy() resourceNew.Spec.FeastProject = "changed" + resourceNew.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + DeploymentStrategy: &appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + } err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -285,6 +291,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) testConfig.Project = resourceNew.Spec.FeastProject + Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) @@ -606,7 +613,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -628,7 +635,7 @@ var _ = Describe("FeatureStore Controller", func() { // check offline config Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -647,7 +654,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(3)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) @@ -795,7 +802,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(3)) Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) @@ -819,7 +826,7 @@ var _ = Describe("FeatureStore Controller", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) - onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(3)) Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) @@ -1033,6 +1040,26 @@ var _ = Describe("FeatureStore Controller", func() { } Expect(repoConfigClient).To(Equal(clientConfig)) + // disable init containers + resource.Spec.Services.DisableInitContainers = true + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: nsName, + }) + Expect(err).NotTo(HaveOccurred()) + + deploy = &appsv1.Deployment{} + objMeta = feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(BeEmpty()) + + // break remote reference hostname := "test:80" referencedRegistry.Spec.Services.Registry = &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index 0b7fe84d22a..06355c664c3 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -246,7 +246,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -267,7 +267,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(repoConfig).To(Equal(&testConfig)) // check offline config - offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer := services.GetOfflineContainer(*deploy) Expect(offlineContainer.Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -284,7 +284,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -388,7 +388,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) // check offline config - offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + offlineContainer = services.GetOfflineContainer(*deploy) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -410,7 +410,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + onlineContainer = services.GetOnlineContainer(*deploy) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 9138f00f2f6..1e5909b8892 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -55,12 +55,13 @@ var _ = Describe("Repo Config", func() { By("Having the local registry resource") featureStore = minimalFeatureStore() + testPath := "/test/file.db" featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ Registry: &feastdevv1alpha1.Registry{ Local: &feastdevv1alpha1.LocalRegistryConfig{ Persistence: &feastdevv1alpha1.RegistryPersistence{ FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{ - Path: "file.db", + Path: testPath, }, }, }, @@ -70,7 +71,7 @@ var _ = Describe("Repo Config", func() { expectedRegistryConfig = RegistryConfig{ RegistryType: "file", - Path: "file.db", + Path: testPath, } repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) @@ -80,53 +81,46 @@ var _ = Describe("Repo Config", func() { Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) - By("Having the remote registry resource") - featureStore = minimalFeatureStore() - featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - Registry: &feastdevv1alpha1.Registry{ - Remote: &feastdevv1alpha1.RemoteRegistryConfig{ - FeastRef: &feastdevv1alpha1.FeatureStoreRef{ - Name: "registry", + By("Adding an offlineStore with PVC") + featureStore.Spec.Services.OfflineStore = &feastdevv1alpha1.OfflineStore{ + Persistence: &feastdevv1alpha1.OfflineStorePersistence{ + FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ + PvcConfig: &feastdevv1alpha1.PvcConfig{ + MountPath: "/testing", }, }, }, } ApplyDefaultsToStatus(featureStore) + appliedServices := featureStore.Status.Applied.Services + Expect(appliedServices.OnlineStore).To(BeNil()) + Expect(appliedServices.OnlineStore).To(BeNil()) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.OfflineStore).To(Equal(defaultOfflineStoreConfig)) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) - - By("Having an offlineStore with PVC") - mountPath := "/testing" - expectedOnlineConfig.Path = mountPath + "/" + DefaultOnlineStorePath - expectedRegistryConfig.Path = mountPath + "/" + DefaultRegistryPath + By("Having the remote registry resource") featureStore = minimalFeatureStore() featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{ - Persistence: &feastdevv1alpha1.OfflineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ - PvcConfig: &feastdevv1alpha1.PvcConfig{ - MountPath: mountPath, - }, + Registry: &feastdevv1alpha1.Registry{ + Remote: &feastdevv1alpha1.RemoteRegistryConfig{ + FeastRef: &feastdevv1alpha1.FeatureStoreRef{ + Name: "registry", }, }, }, } ApplyDefaultsToStatus(featureStore) - appliedServices := featureStore.Status.Applied.Services - Expect(appliedServices.OnlineStore).To(BeNil()) - Expect(appliedServices.Registry.Local.Persistence.FilePersistence.Path).To(Equal(expectedRegistryConfig.Path)) - repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(defaultOfflineStoreConfig)) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) By("Having the all the file services") featureStore = minimalFeatureStore() diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 16f3e663902..1c8a7d26e53 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -49,6 +49,9 @@ func (feast *FeastServices) ApplyDefaults() error { // Deploy the feast services func (feast *FeastServices) Deploy() error { + if feast.noLocalServiceConfigured() { + return errors.New("At least one local service must be configured. e.g. registry / online / offline.") + } openshiftTls, err := feast.checkOpenshiftTls() if err != nil { return err @@ -278,9 +281,7 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { deploy.Spec = appsv1.DeploymentSpec{ Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RecreateDeploymentStrategyType, - }, + Strategy: feast.getDeploymentStrategy(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deploy.GetLabels(), @@ -313,24 +314,9 @@ func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { if err != nil { return err } - feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject - workingDir := getOfflineMountPath(feast.Handler.FeatureStore) - podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ - Name: "feast-init", - Image: DefaultImage, - Env: []corev1.EnvVar{ - { - Name: TmpFeatureStoreYamlEnvVar, - Value: fsYamlB64, - }, - }, - Command: []string{"/bin/sh", "-c"}, - Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + - feastProject + " ] || feast init " + feastProject + ";\necho $" + - TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastProject + - "/feature_repo/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, - WorkingDir: workingDir, - }) + + feast.setInitContainer(podSpec, fsYamlB64) + if feast.isLocalRegistry() { feast.setContainer(&podSpec.Containers, RegistryFeastType, fsYamlB64) } @@ -351,7 +337,7 @@ func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastTy container := &corev1.Container{ Name: string(feastType), Image: *defaultServiceConfigs.Image, - WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "/feature_repo", + WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + FeatureRepoDir, Command: feast.getContainerCommand(feastType), Ports: []corev1.ContainerPort{ { @@ -365,12 +351,6 @@ func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastTy Name: TmpFeatureStoreYamlEnvVar, Value: fsYamlB64, }, - /* - { - Name: mlpConfigVar, - Value: DefaultMlpConfigPath, - }, - */ }, StartupProbe: &corev1.Probe{ ProbeHandler: probeHandler, @@ -417,26 +397,62 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st return feastCommand } -func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { - hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry - // add grpc init container if remote registry reference (feastRef) is configured - if len(hostname) > 0 && feast.IsRemoteRefRegistry() { - grpcurlFlag := "-plaintext" - hostSplit := strings.Split(hostname, ":") - if len(hostSplit) > 1 && hostSplit[1] == "443" { - grpcurlFlag = "-insecure" - } +func (feast *FeastServices) getDeploymentStrategy() appsv1.DeploymentStrategy { + if feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy != nil { + return *feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy + } + return appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + } +} + +func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 string) { + if !feast.Handler.FeatureStore.Status.Applied.Services.DisableInitContainers { + feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject + feastRepoDir := feastProject + FeatureRepoDir + workingDir := getOfflineMountPath(feast.Handler.FeatureStore) podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ - Name: "init-registry", - Image: "fullstorydev/grpcurl:v1.9.1-alpine", - Command: []string{ - "sh", "-c", - "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + - grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", + Name: "feast-init", + Image: getFeatureServerImage(), + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + + feastRepoDir + " ] || feast init " + feastProject + ";\necho $" + + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastRepoDir + + "/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + WorkingDir: workingDir, }) } } + +// add grpc init container if remote registry reference (feastRef) is configured +func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { + if !feast.Handler.FeatureStore.Status.Applied.Services.DisableInitContainers { + hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry + if len(hostname) > 0 && feast.IsRemoteRefRegistry() { + grpcurlFlag := "-plaintext" + hostSplit := strings.Split(hostname, ":") + if len(hostSplit) > 1 && hostSplit[1] == "443" { + grpcurlFlag = "-insecure" + } + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "init-registry", + Image: getGrpcCurlImage(), + Command: []string{ + "sh", "-c", + "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", + }, + }) + } + } +} + func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { svc.Labels = feast.getFeastTypeLabels(feastType) if feast.isOpenShiftTls(feastType) { @@ -632,6 +648,9 @@ func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, err } return nil, err } + if feast.Handler.FeatureStore.Status.Applied.FeastProject != remoteFeastObj.Status.Applied.FeastProject { + return nil, errors.New("FeatureStore '" + remoteFeastObj.Name + "' is using a different feast project than '" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "'. Project names must match.") + } return &FeastServices{ Handler: handler.FeastHandler{ Client: feast.Handler.Client, @@ -672,6 +691,10 @@ func (feast *FeastServices) isOnlinStore() bool { return appliedServices != nil && appliedServices.OnlineStore != nil } +func (feast *FeastServices) noLocalServiceConfigured() bool { + return !(feast.isLocalRegistry() || feast.isOnlinStore() || feast.isOfflinStore()) +} + func (feast *FeastServices) initFeastDeploy() *appsv1.Deployment { deploy := &appsv1.Deployment{ ObjectMeta: feast.GetObjectMeta(), diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 882839a429f..55a5a42e2a7 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -28,6 +28,7 @@ const ( TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" FeatureStoreYamlCmKey = "feature_store.yaml" EphemeralPath = "/feast-data" + FeatureRepoDir = "/feature_repo" DefaultRegistryPath = "registry.db" DefaultOnlineStorePath = "online_store.db" svcDomain = ".svc.cluster.local" diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index 6dcca7edea1..c1b07e7e1e6 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -190,7 +190,7 @@ func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec * }, }, }) - if i, container := getContainerByType(feastType, podSpec.Containers); container != nil { + if i, container := getContainerByType(feastType, *podSpec); container != nil { podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ Name: volName, MountPath: GetTlsPath(feastType), diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 5baeb381d7f..6c964084204 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -254,15 +254,15 @@ var _ = Describe("TLS Config", func() { err = feast.setDeployment(feastDeploy) Expect(err).ToNot(HaveOccurred()) Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) - Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(GetOfflineContainer(*feastDeploy)).NotTo(BeNil()) Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) - Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).To(ContainElements(ContainSubstring("--key"))) - Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) - Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(GetRegistryContainer(*feastDeploy).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetRegistryContainer(*feastDeploy).VolumeMounts).To(HaveLen(1)) + Expect(GetOfflineContainer(*feastDeploy).Command).To(ContainElements(ContainSubstring("--key"))) + Expect(GetOfflineContainer(*feastDeploy).VolumeMounts).To(HaveLen(2)) + Expect(GetOnlineContainer(*feastDeploy).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetOnlineContainer(*feastDeploy).VolumeMounts).To(HaveLen(1)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 7b9c177c89d..b39a851c14b 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -2,12 +2,14 @@ package services import ( "fmt" + "os" "reflect" "slices" "strings" "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -64,8 +66,12 @@ func shouldCreatePvc(featureStore *feastdevv1alpha1.FeatureStore, feastType Feas } func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { - _, ok := hasPvcConfig(featureStore, OfflineFeastType) - return !ok + for _, feastType := range feastServerTypes { + if _, ok := hasPvcConfig(featureStore, feastType); !ok { + return true + } + } + return false } func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { @@ -160,8 +166,23 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { if defaultConfigs.Image == nil { - defaultConfigs.Image = &DefaultImage + img := getFeatureServerImage() + defaultConfigs.Image = &img + } +} + +func getFeatureServerImage() string { + if img, exists := os.LookupEnv("RELATED_IMAGE_FEATURE_SERVER"); exists { + return img } + return DefaultImage +} + +func getGrpcCurlImage() string { + if img, exists := os.LookupEnv("RELATED_IMAGE_GRPC_CURL"); exists { + return img + } + return "fullstorydev/grpcurl:v1.9.1-alpine" } func checkOfflineStoreFilePersistenceType(value string) error { @@ -204,16 +225,16 @@ func defaultOnlineStorePath(featureStore *feastdevv1alpha1.FeatureStore) string if _, ok := hasPvcConfig(featureStore, OnlineFeastType); ok { return DefaultOnlineStorePath } - // if online pvc not set, use offline's mount path. - return getOfflineMountPath(featureStore) + "/" + DefaultOnlineStorePath + // if pvc not set, use the ephemeral mount path. + return EphemeralPath + "/" + DefaultOnlineStorePath } func defaultRegistryPath(featureStore *feastdevv1alpha1.FeatureStore) string { if _, ok := hasPvcConfig(featureStore, RegistryFeastType); ok { return DefaultRegistryPath } - // if registry pvc not set, use offline's mount path. - return getOfflineMountPath(featureStore) + "/" + DefaultRegistryPath + // if pvc not set, use the ephemeral mount path. + return EphemeralPath + "/" + DefaultRegistryPath } func checkOfflineStoreDBStorePersistenceType(value string) error { @@ -238,15 +259,14 @@ func checkRegistryDBStorePersistenceType(value string) error { } func (feast *FeastServices) getSecret(secretRef string) (*corev1.Secret, error) { + logger := log.FromContext(feast.Handler.Context) secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretRef, Namespace: feast.Handler.FeatureStore.Namespace}} objectKey := client.ObjectKeyFromObject(secret) if err := feast.Handler.Client.Get(feast.Handler.Context, objectKey, secret); err != nil { - if apierrors.IsNotFound(err) || err != nil { - logger := log.FromContext(feast.Handler.Context) + if apierrors.IsNotFound(err) { logger.Error(err, "invalid secret "+secretRef+" for offline store") - - return nil, err } + return nil, err } return secret, nil @@ -366,23 +386,23 @@ func envOverride(dst, src []corev1.EnvVar) []corev1.EnvVar { return dst } -func GetRegistryContainer(containers []corev1.Container) *corev1.Container { - _, container := getContainerByType(RegistryFeastType, containers) +func GetRegistryContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(RegistryFeastType, deployment.Spec.Template.Spec) return container } -func GetOfflineContainer(containers []corev1.Container) *corev1.Container { - _, container := getContainerByType(OfflineFeastType, containers) +func GetOfflineContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(OfflineFeastType, deployment.Spec.Template.Spec) return container } -func GetOnlineContainer(containers []corev1.Container) *corev1.Container { - _, container := getContainerByType(OnlineFeastType, containers) +func GetOnlineContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(OnlineFeastType, deployment.Spec.Template.Spec) return container } -func getContainerByType(feastType FeastServiceType, containers []corev1.Container) (int, *corev1.Container) { - for i, c := range containers { +func getContainerByType(feastType FeastServiceType, podSpec corev1.PodSpec) (int, *corev1.Container) { + for i, c := range podSpec.Containers { if c.Name == string(feastType) { return i, &c } diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 8dabe1104f5..0e0ac0bce6d 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -14,6 +14,7 @@ infra/feast-helm-operator/Makefile 6 infra/feast-helm-operator/config/manager/kustomization.yaml 8 infra/feast-operator/Makefile 6 infra/feast-operator/config/manager/kustomization.yaml 8 +infra/feast-operator/config/default/manager_related_images_patch.yaml 5 infra/feast-operator/config/overlays/odh/params.env 1 infra/feast-operator/api/feastversion/version.go 20 java/pom.xml 38 From fc8cff0ded7daa41bdfcbe793bc73e8a045d539f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Jan 2025 22:53:33 -0500 Subject: [PATCH 88/90] chore: Bump jinja2 from 3.1.4 to 3.1.5 in /sdk/python/requirements (#4876) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.5) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index c0b1c348a3a..af41030a6de 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -362,7 +362,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 87b9cf04c91..2488e12304b 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -53,7 +53,7 @@ idna==3.10 # requests importlib-metadata==8.5.0 # via dask -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 37bcbdb2c9c..10c31f30953 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -353,7 +353,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c536ef91ae3..6919925f32b 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -51,7 +51,7 @@ idna==3.10 # requests importlib-metadata==8.5.0 # via dask -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index da914388de2..b0e00409f18 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -371,7 +371,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 80f1e499e1f..aab78c8ceef 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -55,7 +55,7 @@ importlib-metadata==8.5.0 # via # dask # typeguard -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) From d3495a09083b1e6a746fff8444f0bbb887d6ac8b Mon Sep 17 00:00:00 2001 From: Dharmisha Doshi Date: Sun, 19 Jan 2025 06:36:26 -0800 Subject: [PATCH 89/90] fix: Resolving syntax error while querying a feature view with column name starting with a number and BigQuery as data source (#4908) * Fixing issue https://github.com/feast-dev/feast/issues/4688 Signed-off-by: Dharmisha Doshi * Fixing issue https://github.com/feast-dev/feast/issues/4688 Signed-off-by: Dharmisha Doshi * Fixing issue https://github.com/feast-dev/feast/issues/4688 Signed-off-by: Dharmisha Doshi * resolving PEP 8 warnings Signed-off-by: Dharmisha Doshi * resolving linting issues Signed-off-by: Dharmisha Doshi --------- Signed-off-by: Dharmisha Doshi Co-authored-by: n0g0791 Co-authored-by: nishantgaurav-dev Co-authored-by: dharmishadoshi@gmail.com --- sdk/python/feast/infra/offline_stores/bigquery.py | 9 ++++++--- .../feast/infra/offline_stores/offline_utils.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 23f80d79ff2..f0516b594ee 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -901,7 +901,10 @@ def arrow_schema_to_bq_schema(arrow_schema: pyarrow.Schema) -> List[SchemaField] {{ featureview.created_timestamp_column ~ ' as created_timestamp,' if featureview.created_timestamp_column else '' }} {{ featureview.entity_selections | join(', ')}}{% if featureview.entity_selections %},{% else %}{% endif %} {% for feature in featureview.features %} - {{ feature }} as {% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %}{% if loop.last %}{% else %}, {% endif %} + {{ feature | backticks }} as {% if full_feature_names %} + {{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %} + {{ featureview.field_mapping.get(feature, feature) | backticks }}{% endif %} + {% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} WHERE {{ featureview.timestamp_field }} <= '{{ featureview.max_event_timestamp }}' @@ -995,14 +998,14 @@ def arrow_schema_to_bq_schema(arrow_schema: pyarrow.Schema) -> List[SchemaField] The entity_dataframe dataset being our source of truth here. */ -SELECT {{ final_output_feature_names | join(', ')}} +SELECT {{ final_output_feature_names | backticks | join(', ')}} FROM entity_dataframe {% for featureview in featureviews %} LEFT JOIN ( SELECT {{featureview.name}}__entity_row_unique_id {% for feature in featureview.features %} - ,{% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %} + ,{% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) | backticks }}{% endif %} {% endfor %} FROM {{ featureview.name }}__cleaned ) USING ({{featureview.name}}__entity_row_unique_id) diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 2d4fa268e40..2076f977acc 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -186,7 +186,9 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for Bigquery and Redshift""" - template = Environment(loader=BaseLoader()).from_string(source=query_template) + env = Environment(loader=BaseLoader()) + env.filters["backticks"] = enclose_in_backticks + template = env.from_string(source=query_template) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( @@ -252,3 +254,11 @@ def get_pyarrow_schema_from_batch_source( column_names.append(column_name) return pa.schema(pa_schema), column_names + + +def enclose_in_backticks(value): + # Check if the input is a list + if isinstance(value, list): + return [f"`{v}`" for v in value] + else: + return f"`{value}`" From 5a7e6f8a9db3ca77ebf625cb1b9dce57b126d980 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Mon, 20 Jan 2025 15:11:53 +0000 Subject: [PATCH 90/90] chore(release): release 0.43.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # [0.43.0](https://github.com/feast-dev/feast/compare/v0.42.0...v0.43.0) (2025-01-20) ### Bug Fixes * Add k8s module to feature-server image ([#4839](https://github.com/feast-dev/feast/issues/4839)) ([f565565](https://github.com/feast-dev/feast/commit/f565565e0132ea5170221dc6af2e93a5dc3e750d)) * Adding input to workflow ([e3e8c97](https://github.com/feast-dev/feast/commit/e3e8c975b4b9891913d0be8d50df909d4d243191)) * Change image push to use --all-tags option ([#4926](https://github.com/feast-dev/feast/issues/4926)) ([02458fd](https://github.com/feast-dev/feast/commit/02458fd7aad49d5daa5b9836f5abdc4dd81d07bb)) * Fix integration build/push for images ([#4923](https://github.com/feast-dev/feast/issues/4923)) ([695e49b](https://github.com/feast-dev/feast/commit/695e49bd93a4c8af2ce5839586295b5e74e1b98e)) * Fix integration operator push ([#4924](https://github.com/feast-dev/feast/issues/4924)) ([13c7267](https://github.com/feast-dev/feast/commit/13c7267b555cca4f3361f34fb384a6fd9f27dedf)) * Fix release.yml ([#4845](https://github.com/feast-dev/feast/issues/4845)) ([b4768a8](https://github.com/feast-dev/feast/commit/b4768a81b94352de037dc305df309fcf06fd2973)) * Fixing some of the warnings with the github actions ([#4763](https://github.com/feast-dev/feast/issues/4763)) ([1119439](https://github.com/feast-dev/feast/commit/1119439c49bc90e62f02da078901509c1d740236)) * Improve status.applied updates & add offline pvc unit test ([#4871](https://github.com/feast-dev/feast/issues/4871)) ([3f49517](https://github.com/feast-dev/feast/commit/3f49517dfeabea5ffbd3f6b589cc0f2280ee4018)) * Made fixes to Go Operator DB persistence ([#4830](https://github.com/feast-dev/feast/issues/4830)) ([cdc0753](https://github.com/feast-dev/feast/commit/cdc075360242bfdf3812d394a3c9c550f81b0f98)) * Make transformation_service_endpoint configuration optional ([#4880](https://github.com/feast-dev/feast/issues/4880)) ([c62377b](https://github.com/feast-dev/feast/commit/c62377bc095a83022d13e5a8a3a9413d7e0f3e2c)) * Move pre-release image builds to quay.io, retire gcr.io pushes ([#4922](https://github.com/feast-dev/feast/issues/4922)) ([40b975b](https://github.com/feast-dev/feast/commit/40b975b8468de2678af8b191e93495e51af0b6aa)) * Performance regression in /get-online-features ([#4892](https://github.com/feast-dev/feast/issues/4892)) ([0db56a2](https://github.com/feast-dev/feast/commit/0db56a2cb5888bc21dbdb331e2b5fc3d33508424)) * Refactor Operator to deploy all feast services to the same Deployment/Pod ([#4863](https://github.com/feast-dev/feast/issues/4863)) ([88854dd](https://github.com/feast-dev/feast/commit/88854dd56fd0becf4a5d5293735a1c9ba394d53d)) * Remove unnecessary google cloud steps & upgrade docker action versions ([#4925](https://github.com/feast-dev/feast/issues/4925)) ([32aaf9a](https://github.com/feast-dev/feast/commit/32aaf9aba96c53e1c69577312982472182e99659)) * Remove verifyClient TLS offlineStore option from the Operator ([#4847](https://github.com/feast-dev/feast/issues/4847)) ([79fa247](https://github.com/feast-dev/feast/commit/79fa247026dd95e75a19308d437997310d061b35)) * Resolving syntax error while querying a feature view with column name starting with a number and BigQuery as data source ([#4908](https://github.com/feast-dev/feast/issues/4908)) ([d3495a0](https://github.com/feast-dev/feast/commit/d3495a09083b1e6a746fff8444f0bbb887d6ac8b)) * Updated python-helm-demo example to use MinIO instead of GS ([#4691](https://github.com/feast-dev/feast/issues/4691)) ([31afd99](https://github.com/feast-dev/feast/commit/31afd99c0969002fe04982e40cf7a857960f7abf)) ### Features * Add date field support to spark ([#4913](https://github.com/feast-dev/feast/issues/4913)) ([a8aeb79](https://github.com/feast-dev/feast/commit/a8aeb79830f12358c2355be44fca68e61992cb46)) * Add date support when converting from python to feast types ([#4918](https://github.com/feast-dev/feast/issues/4918)) ([bd9f071](https://github.com/feast-dev/feast/commit/bd9f071017756e205fbabe6af0d38dfaa9be3d7b)) * Add duckdb extra to multicloud release image ([#4862](https://github.com/feast-dev/feast/issues/4862)) ([b539eba](https://github.com/feast-dev/feast/commit/b539ebaad5ec2c1a199fe08ceccd206754ce82f0)) * Add milvus package to release image & option to Operator ([#4870](https://github.com/feast-dev/feast/issues/4870)) ([ef724b6](https://github.com/feast-dev/feast/commit/ef724b66bd4d5f355b055d6d81525c4a17ce94c1)) * Add Milvus Vector Database Implementation ([#4751](https://github.com/feast-dev/feast/issues/4751)) ([22c7b58](https://github.com/feast-dev/feast/commit/22c7b58f9590a357eaa57c77d5ed351f1fa07501)) * Add online/offline replica support ([#4812](https://github.com/feast-dev/feast/issues/4812)) ([b97da6c](https://github.com/feast-dev/feast/commit/b97da6ca3a08e3f0fc35552dd7f0bd3b59083f35)) * Added pvc accessModes support ([#4851](https://github.com/feast-dev/feast/issues/4851)) ([a73514c](https://github.com/feast-dev/feast/commit/a73514cd4f7fecbc89679566e0f8a0af16b6b06d)) * Adding EnvFrom support for the OptionalConfigs type to the Go Operator ([#4909](https://github.com/feast-dev/feast/issues/4909)) ([e01e510](https://github.com/feast-dev/feast/commit/e01e51076f5d8fe5be459037bd254e6f94e0cb0f)) * Adding Feature Server to components docs ([#4868](https://github.com/feast-dev/feast/issues/4868)) ([f95e54b](https://github.com/feast-dev/feast/commit/f95e54bdbee80be6b0e290a02e56f92daac2cf64)) * Adding features field to retrieve_online_features to return mor… ([#4869](https://github.com/feast-dev/feast/issues/4869)) ([7df287e](https://github.com/feast-dev/feast/commit/7df287e8c0f5ec3ab3fa88fd5576f636053a3769)) * Adding packages for Milvus Online Store ([#4854](https://github.com/feast-dev/feast/issues/4854)) ([49171bd](https://github.com/feast-dev/feast/commit/49171bd53fb8bfc325eb7167cac8cae18a28bd63)) * Adding vector_search parameter to fields ([#4855](https://github.com/feast-dev/feast/issues/4855)) ([739eaa7](https://github.com/feast-dev/feast/commit/739eaa78e6d995ee0750292d2f8d81886a3f9829)) * Feast Operator support log level configuration for services ([#4808](https://github.com/feast-dev/feast/issues/4808)) ([19424bc](https://github.com/feast-dev/feast/commit/19424bcc975d90d922791b5bd0da6ac13955c0c5)) * Go Operator - Parsing the output to go structs ([#4832](https://github.com/feast-dev/feast/issues/4832)) ([732865f](https://github.com/feast-dev/feast/commit/732865f20e7fae7a46f54be7bc469ce2b3bc44e2)) * Implement `date_partition_column` for `SparkSource` ([#4844](https://github.com/feast-dev/feast/issues/4844)) ([c5ffa03](https://github.com/feast-dev/feast/commit/c5ffa037cb030c64d6e25995199cf762cc0e9b2a)) * Loading the CA trusted store certificate into Feast to verify the public certificate. ([#4852](https://github.com/feast-dev/feast/issues/4852)) ([132ce2a](https://github.com/feast-dev/feast/commit/132ce2a6c9e3ff8544680d5237e9e1523d988d7e)) * Operator E2E test to validate FeatureStore custom resource using remote registry ([#4822](https://github.com/feast-dev/feast/issues/4822)) ([d558ef7](https://github.com/feast-dev/feast/commit/d558ef7e19aa561c37c38d4d0da2b8c1467414f5)) * Operator improvements ([#4928](https://github.com/feast-dev/feast/issues/4928)) ([7a1f4dd](https://github.com/feast-dev/feast/commit/7a1f4dd8b96a40d055467e1e5f72c91167e40484)) * Removing the tls_verify_client flag from feast cli for offline server. ([#4842](https://github.com/feast-dev/feast/issues/4842)) ([8320e23](https://github.com/feast-dev/feast/commit/8320e23eb85cc419ef8aa0fdc07efa81857e0345)) * Separating the RBAC and Remote related integration tests. ([#4905](https://github.com/feast-dev/feast/issues/4905)) ([76e1e21](https://github.com/feast-dev/feast/commit/76e1e2178c285886136e8f2fc4436302e4291715)) * Snyk vulnerability issues fix. ([#4867](https://github.com/feast-dev/feast/issues/4867)) ([dbc9207](https://github.com/feast-dev/feast/commit/dbc92070c8ef6b9e4e53d89ec03090bf30bd0f60)), closes [#6](https://github.com/feast-dev/feast/issues/6) [#3](https://github.com/feast-dev/feast/issues/3) [#4](https://github.com/feast-dev/feast/issues/4) * Use ASOF JOIN in Snowflake offline store query ([#4850](https://github.com/feast-dev/feast/issues/4850)) ([8f591a2](https://github.com/feast-dev/feast/commit/8f591a235ba5bd9d1bc598195f46c7e12e437a2c)) ### Reverts * Revert "chore: Add Milvus to pr_integration_tests.yml" ([#4900](https://github.com/feast-dev/feast/issues/4900)) ([07958f7](https://github.com/feast-dev/feast/commit/07958f71cd89984325ec3ca2006b17fe5d333d02)), closes [#4891](https://github.com/feast-dev/feast/issues/4891) --- CHANGELOG.md | 54 +++++++++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 8 +-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +-- .../feast/charts/feature-server/Chart.yaml | 4 +- .../feast/charts/feature-server/README.md | 4 +- .../feast/charts/feature-server/values.yaml | 2 +- .../charts/transformation-service/Chart.yaml | 4 +- .../charts/transformation-service/README.md | 4 +- .../charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 +- infra/feast-helm-operator/Makefile | 2 +- .../config/manager/kustomization.yaml | 2 +- infra/feast-operator/Makefile | 2 +- .../api/feastversion/version.go | 2 +- .../default/manager_related_images_patch.yaml | 2 +- .../config/manager/kustomization.yaml | 2 +- .../config/overlays/odh/params.env | 2 +- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 46 +++++++++++++--- ui/package.json | 2 +- 24 files changed, 125 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d97c33480c..93b7ea5edf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +# [0.43.0](https://github.com/feast-dev/feast/compare/v0.42.0...v0.43.0) (2025-01-20) + + +### Bug Fixes + +* Add k8s module to feature-server image ([#4839](https://github.com/feast-dev/feast/issues/4839)) ([f565565](https://github.com/feast-dev/feast/commit/f565565e0132ea5170221dc6af2e93a5dc3e750d)) +* Adding input to workflow ([e3e8c97](https://github.com/feast-dev/feast/commit/e3e8c975b4b9891913d0be8d50df909d4d243191)) +* Change image push to use --all-tags option ([#4926](https://github.com/feast-dev/feast/issues/4926)) ([02458fd](https://github.com/feast-dev/feast/commit/02458fd7aad49d5daa5b9836f5abdc4dd81d07bb)) +* Fix integration build/push for images ([#4923](https://github.com/feast-dev/feast/issues/4923)) ([695e49b](https://github.com/feast-dev/feast/commit/695e49bd93a4c8af2ce5839586295b5e74e1b98e)) +* Fix integration operator push ([#4924](https://github.com/feast-dev/feast/issues/4924)) ([13c7267](https://github.com/feast-dev/feast/commit/13c7267b555cca4f3361f34fb384a6fd9f27dedf)) +* Fix release.yml ([#4845](https://github.com/feast-dev/feast/issues/4845)) ([b4768a8](https://github.com/feast-dev/feast/commit/b4768a81b94352de037dc305df309fcf06fd2973)) +* Fixing some of the warnings with the github actions ([#4763](https://github.com/feast-dev/feast/issues/4763)) ([1119439](https://github.com/feast-dev/feast/commit/1119439c49bc90e62f02da078901509c1d740236)) +* Improve status.applied updates & add offline pvc unit test ([#4871](https://github.com/feast-dev/feast/issues/4871)) ([3f49517](https://github.com/feast-dev/feast/commit/3f49517dfeabea5ffbd3f6b589cc0f2280ee4018)) +* Made fixes to Go Operator DB persistence ([#4830](https://github.com/feast-dev/feast/issues/4830)) ([cdc0753](https://github.com/feast-dev/feast/commit/cdc075360242bfdf3812d394a3c9c550f81b0f98)) +* Make transformation_service_endpoint configuration optional ([#4880](https://github.com/feast-dev/feast/issues/4880)) ([c62377b](https://github.com/feast-dev/feast/commit/c62377bc095a83022d13e5a8a3a9413d7e0f3e2c)) +* Move pre-release image builds to quay.io, retire gcr.io pushes ([#4922](https://github.com/feast-dev/feast/issues/4922)) ([40b975b](https://github.com/feast-dev/feast/commit/40b975b8468de2678af8b191e93495e51af0b6aa)) +* Performance regression in /get-online-features ([#4892](https://github.com/feast-dev/feast/issues/4892)) ([0db56a2](https://github.com/feast-dev/feast/commit/0db56a2cb5888bc21dbdb331e2b5fc3d33508424)) +* Refactor Operator to deploy all feast services to the same Deployment/Pod ([#4863](https://github.com/feast-dev/feast/issues/4863)) ([88854dd](https://github.com/feast-dev/feast/commit/88854dd56fd0becf4a5d5293735a1c9ba394d53d)) +* Remove unnecessary google cloud steps & upgrade docker action versions ([#4925](https://github.com/feast-dev/feast/issues/4925)) ([32aaf9a](https://github.com/feast-dev/feast/commit/32aaf9aba96c53e1c69577312982472182e99659)) +* Remove verifyClient TLS offlineStore option from the Operator ([#4847](https://github.com/feast-dev/feast/issues/4847)) ([79fa247](https://github.com/feast-dev/feast/commit/79fa247026dd95e75a19308d437997310d061b35)) +* Resolving syntax error while querying a feature view with column name starting with a number and BigQuery as data source ([#4908](https://github.com/feast-dev/feast/issues/4908)) ([d3495a0](https://github.com/feast-dev/feast/commit/d3495a09083b1e6a746fff8444f0bbb887d6ac8b)) +* Updated python-helm-demo example to use MinIO instead of GS ([#4691](https://github.com/feast-dev/feast/issues/4691)) ([31afd99](https://github.com/feast-dev/feast/commit/31afd99c0969002fe04982e40cf7a857960f7abf)) + + +### Features + +* Add date field support to spark ([#4913](https://github.com/feast-dev/feast/issues/4913)) ([a8aeb79](https://github.com/feast-dev/feast/commit/a8aeb79830f12358c2355be44fca68e61992cb46)) +* Add date support when converting from python to feast types ([#4918](https://github.com/feast-dev/feast/issues/4918)) ([bd9f071](https://github.com/feast-dev/feast/commit/bd9f071017756e205fbabe6af0d38dfaa9be3d7b)) +* Add duckdb extra to multicloud release image ([#4862](https://github.com/feast-dev/feast/issues/4862)) ([b539eba](https://github.com/feast-dev/feast/commit/b539ebaad5ec2c1a199fe08ceccd206754ce82f0)) +* Add milvus package to release image & option to Operator ([#4870](https://github.com/feast-dev/feast/issues/4870)) ([ef724b6](https://github.com/feast-dev/feast/commit/ef724b66bd4d5f355b055d6d81525c4a17ce94c1)) +* Add Milvus Vector Database Implementation ([#4751](https://github.com/feast-dev/feast/issues/4751)) ([22c7b58](https://github.com/feast-dev/feast/commit/22c7b58f9590a357eaa57c77d5ed351f1fa07501)) +* Add online/offline replica support ([#4812](https://github.com/feast-dev/feast/issues/4812)) ([b97da6c](https://github.com/feast-dev/feast/commit/b97da6ca3a08e3f0fc35552dd7f0bd3b59083f35)) +* Added pvc accessModes support ([#4851](https://github.com/feast-dev/feast/issues/4851)) ([a73514c](https://github.com/feast-dev/feast/commit/a73514cd4f7fecbc89679566e0f8a0af16b6b06d)) +* Adding EnvFrom support for the OptionalConfigs type to the Go Operator ([#4909](https://github.com/feast-dev/feast/issues/4909)) ([e01e510](https://github.com/feast-dev/feast/commit/e01e51076f5d8fe5be459037bd254e6f94e0cb0f)) +* Adding Feature Server to components docs ([#4868](https://github.com/feast-dev/feast/issues/4868)) ([f95e54b](https://github.com/feast-dev/feast/commit/f95e54bdbee80be6b0e290a02e56f92daac2cf64)) +* Adding features field to retrieve_online_features to return mor… ([#4869](https://github.com/feast-dev/feast/issues/4869)) ([7df287e](https://github.com/feast-dev/feast/commit/7df287e8c0f5ec3ab3fa88fd5576f636053a3769)) +* Adding packages for Milvus Online Store ([#4854](https://github.com/feast-dev/feast/issues/4854)) ([49171bd](https://github.com/feast-dev/feast/commit/49171bd53fb8bfc325eb7167cac8cae18a28bd63)) +* Adding vector_search parameter to fields ([#4855](https://github.com/feast-dev/feast/issues/4855)) ([739eaa7](https://github.com/feast-dev/feast/commit/739eaa78e6d995ee0750292d2f8d81886a3f9829)) +* Feast Operator support log level configuration for services ([#4808](https://github.com/feast-dev/feast/issues/4808)) ([19424bc](https://github.com/feast-dev/feast/commit/19424bcc975d90d922791b5bd0da6ac13955c0c5)) +* Go Operator - Parsing the output to go structs ([#4832](https://github.com/feast-dev/feast/issues/4832)) ([732865f](https://github.com/feast-dev/feast/commit/732865f20e7fae7a46f54be7bc469ce2b3bc44e2)) +* Implement `date_partition_column` for `SparkSource` ([#4844](https://github.com/feast-dev/feast/issues/4844)) ([c5ffa03](https://github.com/feast-dev/feast/commit/c5ffa037cb030c64d6e25995199cf762cc0e9b2a)) +* Loading the CA trusted store certificate into Feast to verify the public certificate. ([#4852](https://github.com/feast-dev/feast/issues/4852)) ([132ce2a](https://github.com/feast-dev/feast/commit/132ce2a6c9e3ff8544680d5237e9e1523d988d7e)) +* Operator E2E test to validate FeatureStore custom resource using remote registry ([#4822](https://github.com/feast-dev/feast/issues/4822)) ([d558ef7](https://github.com/feast-dev/feast/commit/d558ef7e19aa561c37c38d4d0da2b8c1467414f5)) +* Operator improvements ([#4928](https://github.com/feast-dev/feast/issues/4928)) ([7a1f4dd](https://github.com/feast-dev/feast/commit/7a1f4dd8b96a40d055467e1e5f72c91167e40484)) +* Removing the tls_verify_client flag from feast cli for offline server. ([#4842](https://github.com/feast-dev/feast/issues/4842)) ([8320e23](https://github.com/feast-dev/feast/commit/8320e23eb85cc419ef8aa0fdc07efa81857e0345)) +* Separating the RBAC and Remote related integration tests. ([#4905](https://github.com/feast-dev/feast/issues/4905)) ([76e1e21](https://github.com/feast-dev/feast/commit/76e1e2178c285886136e8f2fc4436302e4291715)) +* Snyk vulnerability issues fix. ([#4867](https://github.com/feast-dev/feast/issues/4867)) ([dbc9207](https://github.com/feast-dev/feast/commit/dbc92070c8ef6b9e4e53d89ec03090bf30bd0f60)), closes [#6](https://github.com/feast-dev/feast/issues/6) [#3](https://github.com/feast-dev/feast/issues/3) [#4](https://github.com/feast-dev/feast/issues/4) +* Use ASOF JOIN in Snowflake offline store query ([#4850](https://github.com/feast-dev/feast/issues/4850)) ([8f591a2](https://github.com/feast-dev/feast/commit/8f591a235ba5bd9d1bc598195f46c7e12e437a2c)) + + +### Reverts + +* Revert "chore: Add Milvus to pr_integration_tests.yml" ([#4900](https://github.com/feast-dev/feast/issues/4900)) ([07958f7](https://github.com/feast-dev/feast/commit/07958f71cd89984325ec3ca2006b17fe5d333d02)), closes [#4891](https://github.com/feast-dev/feast/issues/4891) + # [0.42.0](https://github.com/feast-dev/feast/compare/v0.41.0...v0.42.0) (2024-12-05) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index a88a067f9d9..7b8e51b3b52 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.42.1 +version: 0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 1c3e17993ff..f8554cb4dff 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.42.0` +Current chart version is `0.43.0` ## Installation @@ -40,7 +40,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.42.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.43.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | @@ -56,9 +56,9 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | readinessProbe.periodSeconds | int | `10` | | | replicaCount | int | `1` | | | resources | object | `{}` | | +| route.enabled | bool | `false` | | | securityContext | object | `{}` | | | service.port | int | `80` | | | service.type | string | `"ClusterIP"` | | | serviceAccount.name | string | `""` | | -| tolerations | list | `[]` | | -| route.enabled | bool | `false` | | \ No newline at end of file +| tolerations | list | `[]` | | \ No newline at end of file diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 9a0d2986631..3bbf9ec2b20 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.42.0 + tag: 0.43.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index c3dddef6f95..58c4ce34fbb 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.42.0 +version: 0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index e2b92ec44c0..4170541543d 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.42.0` +Feature store for machine learning Current chart version is `0.43.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.42.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.42.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.43.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.43.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index ef4282edaee..f1468973eb2 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.42.0 -appVersion: v0.42.0 +version: 0.43.0 +appVersion: v0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 3c447b4aa63..990a1154884 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.42.0](https://img.shields.io/badge/Version-0.42.0-informational?style=flat-square) ![AppVersion: v0.42.0](https://img.shields.io/badge/AppVersion-v0.42.0-informational?style=flat-square) +![Version: 0.43.0](https://img.shields.io/badge/Version-0.43.0-informational?style=flat-square) ![AppVersion: v0.43.0](https://img.shields.io/badge/AppVersion-v0.43.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.42.0"` | Image tag | +| image.tag | string | `"0.43.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index e53d0293bcb..b7098ef5ac4 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.42.0 + tag: 0.43.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 245ad022180..1c19dd8b7d6 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.42.0 -appVersion: v0.42.0 +version: 0.43.0 +appVersion: v0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index a69d2d5911f..27bd8afc609 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.42.0](https://img.shields.io/badge/Version-0.42.0-informational?style=flat-square) ![AppVersion: v0.42.0](https://img.shields.io/badge/AppVersion-v0.42.0-informational?style=flat-square) +![Version: 0.43.0](https://img.shields.io/badge/Version-0.43.0-informational?style=flat-square) ![AppVersion: v0.43.0](https://img.shields.io/badge/AppVersion-v0.43.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.42.0"` | Image tag | +| image.tag | string | `"0.43.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index d765c3d9130..bb2cd8e4ea5 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.42.0 + tag: 0.43.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 41d6dc25c2b..82c6351eaf5 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.42.0 + version: 0.43.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.42.0 + version: 0.43.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-helm-operator/Makefile b/infra/feast-helm-operator/Makefile index 4b1a9ec56a4..e3edf0622f5 100644 --- a/infra/feast-helm-operator/Makefile +++ b/infra/feast-helm-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.42.0 +VERSION ?= 0.43.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-helm-operator/config/manager/kustomization.yaml b/infra/feast-helm-operator/config/manager/kustomization.yaml index ec9247f695d..e4c3f23a824 100644 --- a/infra/feast-helm-operator/config/manager/kustomization.yaml +++ b/infra/feast-helm-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: feastdev/feast-helm-operator - newTag: 0.42.0 + newTag: 0.43.0 diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 46d4451c989..56f388fde65 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.42.0 +VERSION ?= 0.43.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 4a20c4ca592..50fddbada8e 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version -const FeastVersion = "0.42.0" +const FeastVersion = "0.43.0" diff --git a/infra/feast-operator/config/default/manager_related_images_patch.yaml b/infra/feast-operator/config/default/manager_related_images_patch.yaml index 7ad1ab5970f..084de876612 100644 --- a/infra/feast-operator/config/default/manager_related_images_patch.yaml +++ b/infra/feast-operator/config/default/manager_related_images_patch.yaml @@ -2,7 +2,7 @@ path: "/spec/template/spec/containers/0/env/0" value: name: RELATED_IMAGE_FEATURE_SERVER - value: docker.io/feastdev/feature-server:0.42.0 + value: docker.io/feastdev/feature-server:0.43.0 - op: replace path: "/spec/template/spec/containers/0/env/1" value: diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index 2e0a046bf15..39b762031d5 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: feastdev/feast-operator - newTag: 0.42.0 + newTag: 0.43.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index 16c8e1a6fb0..df62943fbcf 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1 +1 @@ -odh-feast-operator-controller-image=docker.io/feastdev/feast-operator:0.42.0 +odh-feast-operator-controller-image=docker.io/feastdev/feast-operator:0.43.0 diff --git a/java/pom.xml b/java/pom.xml index 82c0a00ba22..8173385e0c9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.42.0 + 0.43.0 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index f1e28382da6..16e413a0002 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.42.0", + "@feast-dev/feast-ui": "0.43.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 6578b4a34f7..b269c2e7f51 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1570,10 +1570,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.42.0": - version "0.42.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.42.0.tgz#b186142d6b5176c8d5784c425fa22724b16dda6f" - integrity sha512-onHkZznObLCy5kpeWv+8Z6O51WMqF7xxNfnn3SNSOj6sPJn+FpfMq5DmJG2ESRvB3lw/SrBJB1aPr+pOtMYQjQ== +"@feast-dev/feast-ui@0.43.0": + version "0.43.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.43.0.tgz#0228a13d3899d8a30109bc07117c5bf141dcc810" + integrity sha512-+IjDLSN+zeY62EmstbtS0JwJKnKJVEJJCGBluktlMHBdCLdoC/Pxs8DDFLd/Q94zMmzM+3kUdaj5zABs2QinWA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^95.12.0" @@ -1587,9 +1587,9 @@ react-app-polyfill "^3.0.0" react-code-blocks "^0.1.6" react-query "^3.39.3" - react-router-dom "<6.4.0" + react-router-dom "^6.28.0" tslib "^2.3.1" - use-query-params "^1.2.3" + use-query-params "^2.2.1" zod "^3.11.6" "@hello-pangea/dnd@^16.6.0": @@ -2056,6 +2056,11 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@remix-run/router@1.21.1": + version "1.21.1" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.21.1.tgz#bf15274d3856c395402719fa6b1dc8cc5245aaf7" + integrity sha512-KeBYSwohb8g4/wCcnksvKTYlg69O62sQeLynn2YE+5z7JWEj95if27kclW9QqbrlsQ2DINI8fjbV3zyuKfwjKg== + "@rollup/plugin-babel@^5.2.0": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -9184,7 +9189,7 @@ react-remove-scroll@^2.6.0: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@6, react-router-dom@<6.4.0: +react-router-dom@6: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== @@ -9192,6 +9197,21 @@ react-router-dom@6, react-router-dom@<6.4.0: history "^5.2.0" react-router "6.3.0" +react-router-dom@^6.28.0: + version "6.28.2" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.28.2.tgz#9bc4f58b0cfe91d39d1a6be4beb0ef051ca9b06e" + integrity sha512-O81EWqNJWqvlN/a7eTudAdQm0TbI7hw+WIi7OwwMcTn5JMyZ0ibTFNGz+t+Lju0df4LcqowCegcrK22lB1q9Kw== + dependencies: + "@remix-run/router" "1.21.1" + react-router "6.28.2" + +react-router@6.28.2: + version "6.28.2" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.28.2.tgz#1ddea57c2de0d99e12d00af14d1499703f1378a9" + integrity sha512-BgFY7+wEGVjHCiqaj2XiUBQ1kkzfg6UoKYwEe0wv+FF+HNPCxtS/MVPvLAPH++EsuCMReZl9RYVGqcHLk5ms3A== + dependencies: + "@remix-run/router" "1.21.1" + react-router@6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557" @@ -9829,6 +9849,11 @@ serialize-query-params@^1.3.5: resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-1.3.6.tgz#5dd5225db85ce747fe6fbc4897628504faafec6d" integrity sha512-VlH7sfWNyPVZClPkRacopn6sn5uQMXBsjPVz1+pBHX895VpcYVznfJtZ49e6jymcrz+l/vowkepCZn/7xEAEdw== +serialize-query-params@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-2.0.2.tgz#598a3fb9e13f4ea1c1992fbd20231aa16b31db81" + integrity sha512-1chMo1dST4pFA9RDXAtF0Rbjaut4is7bzFbI1Z26IuMub68pNCILku85aYmeFhvnY//BXUPUhoRMjYcsT93J/Q== + serve-index@^1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/serve-index/-/serve-index-1.9.1.tgz#d3768d69b1e7d82e5ce050fff5b453bea12a9239" @@ -10874,6 +10899,13 @@ use-query-params@^1.2.3: dependencies: serialize-query-params "^1.3.5" +use-query-params@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-2.2.1.tgz#c558ab70706f319112fbccabf6867b9f904e947d" + integrity sha512-i6alcyLB8w9i3ZK3caNftdb+UnbfBRNPDnc89CNQWkGRmDrm/gfydHvMBfVsQJRq3NoHOM2dt/ceBWG2397v1Q== + dependencies: + serialize-query-params "^2.0.2" + use-sidecar@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2" diff --git a/ui/package.json b/ui/package.json index ea911342091..e057b0e5e42 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.42.0", + "version": "0.43.0", "private": false, "files": [ "dist"