From 2089f9da0bf6f18544722e0d4d269d0ceb63e740 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sat, 14 Mar 2026 15:32:56 +0530 Subject: [PATCH 1/4] feat: Add feast apply init container to automate registry population on pod start Signed-off-by: ntkathole --- .secrets.baseline | 18 +++++++++--------- .../api/v1/featurestore_types.go | 2 ++ .../api/v1/zz_generated.deepcopy.go | 5 +++++ .../api/v1alpha1/featurestore_types.go | 2 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ .../crd/bases/feast.dev_featurestores.yaml | 18 ++++++++++++++++++ infra/feast-operator/dist/install.yaml | 18 ++++++++++++++++++ infra/feast-operator/docs/api/markdown/ref.md | 1 + .../featurestore_controller_cronjob_test.go | 2 ++ ...featurestore_controller_objectstore_test.go | 2 +- .../featurestore_controller_oidc_auth_test.go | 2 +- .../controller/featurestore_controller_test.go | 10 ++++++---- .../internal/controller/services/services.go | 10 ++++++++++ .../internal/controller/services/tls_test.go | 2 +- .../internal/controller/services/util.go | 3 +++ 15 files changed, 84 insertions(+), 16 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index c1f49e8ec91..79aaf6edc4a 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -934,7 +934,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 727 + "line_number": 729 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -943,21 +943,21 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 681 + "line_number": 686 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1249 + "line_number": 1254 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1254 + "line_number": 1259 } ], "infra/feast-operator/api/v1alpha1/featurestore_types.go": [ @@ -966,7 +966,7 @@ "filename": "infra/feast-operator/api/v1alpha1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 647 + "line_number": 649 } ], "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go": [ @@ -975,21 +975,21 @@ "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 590 + "line_number": 595 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1098 + "line_number": 1103 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1103 + "line_number": 1108 } ], "infra/feast-operator/config/samples/v1_featurestore_db_persistence.yaml": [ @@ -1539,5 +1539,5 @@ } ] }, - "generated_at": "2026-03-10T18:11:57Z" + "generated_at": "2026-03-14T16:01:28Z" } diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index 026e5f4a92c..dd12a94f7cd 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -310,6 +310,8 @@ type FeatureStoreServices struct { SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` // Disable the 'feast repo initialization' initContainer DisableInitContainers bool `json:"disableInitContainers,omitempty"` + // Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. + RunFeastApplyOnInit *bool `json:"runFeastApplyOnInit,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). Volumes []corev1.Volume `json:"volumes,omitempty"` // Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 63500266f02..87043017c71 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -369,6 +369,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { *out = new(corev1.PodSecurityContext) (*in).DeepCopyInto(*out) } + if in.RunFeastApplyOnInit != nil { + in, out := &in.RunFeastApplyOnInit, &out.RunFeastApplyOnInit + *out = new(bool) + **out = **in + } if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes *out = make([]corev1.Volume, len(*in)) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 23af949390c..d9c85c93136 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -289,6 +289,8 @@ type FeatureStoreServices struct { SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` // Disable the 'feast repo initialization' initContainer DisableInitContainers bool `json:"disableInitContainers,omitempty"` + // Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. + RunFeastApplyOnInit *bool `json:"runFeastApplyOnInit,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). Volumes []corev1.Volume `json:"volumes,omitempty"` } diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index fa7c6f210dd..4033c368c8b 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -315,6 +315,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { *out = new(v1.PodSecurityContext) (*in).DeepCopyInto(*out) } + if in.RunFeastApplyOnInit != nil { + in, out := &in.RunFeastApplyOnInit, &out.RunFeastApplyOnInit + *out = new(bool) + **out = **in + } if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes *out = make([]v1.Volume, len(*in)) diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index c2f5fc3eb20..2533ae930de 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -3129,6 +3129,10 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the registry. + Defaults to true. Ignored when DisableInitContainers is true. + type: boolean scaling: description: Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -8871,6 +8875,11 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the + registry. Defaults to true. Ignored when DisableInitContainers + is true. + type: boolean scaling: description: Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -13920,6 +13929,10 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the registry. + Defaults to true. Ignored when DisableInitContainers is true. + type: boolean securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -18163,6 +18176,11 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the + registry. Defaults to true. Ignored when DisableInitContainers + is true. + type: boolean securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index f0057be966e..46b42928841 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -3137,6 +3137,10 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the registry. + Defaults to true. Ignored when DisableInitContainers is true. + type: boolean scaling: description: Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -8879,6 +8883,11 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the + registry. Defaults to true. Ignored when DisableInitContainers + is true. + type: boolean scaling: description: Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -13928,6 +13937,10 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the registry. + Defaults to true. Ignored when DisableInitContainers is true. + type: boolean securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -18171,6 +18184,11 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + runFeastApplyOnInit: + description: Runs feast apply on pod start to populate the + registry. Defaults to true. Ignored when DisableInitContainers + is true. + type: boolean securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index 698c3f6bbe3..b7c7a1b8a71 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -240,6 +240,7 @@ _Appears in:_ | `deploymentStrategy` _[DeploymentStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#deploymentstrategy-v1-apps)_ | | | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podsecuritycontext-v1-core)_ | | | `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | +| `runFeastApplyOnInit` _boolean_ | Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. | | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). | | `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). For static replicas, use spec.replicas instead. | diff --git a/infra/feast-operator/internal/controller/featurestore_controller_cronjob_test.go b/infra/feast-operator/internal/controller/featurestore_controller_cronjob_test.go index c329d70f06d..11ae2af7777 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_cronjob_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_cronjob_test.go @@ -98,6 +98,8 @@ var _ = Describe("FeatureStore Controller - Feast CronJob", func() { Expect(resource.Status).NotTo(BeNil()) Expect(resource.Status.CronJob).To(Equal(objMeta.Name)) Expect(resource.Status.Applied.CronJob.Schedule).NotTo(BeEmpty()) + Expect(resource.Status.Applied.Services.RunFeastApplyOnInit).NotTo(BeNil()) + Expect(*resource.Status.Applied.Services.RunFeastApplyOnInit).To(BeTrue()) Expect(resource.Status.Conditions).NotTo(BeEmpty()) cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1.CronJobReadyType) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index 37d22094147..a326752a78f 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -185,7 +185,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(1))) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) Expect(services.GetRegistryContainer(*deploy)).NotTo(BeNil()) Expect(services.GetOnlineContainer(*deploy)).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index bb5cc4fb4b2..b8af9484acc 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -221,7 +221,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(1))) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) Expect(services.GetOfflineContainer(*deploy).VolumeMounts).To(HaveLen(1)) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index a70cd476679..40a66c24ef5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -210,8 +210,10 @@ var _ = Describe("FeatureStore Controller", func() { Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(1))) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init")) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Name).To(Equal("feast-apply")) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Command).To(Equal([]string{"feast", "apply"})) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) deploy.Spec.Replicas = int32Ptr(3) @@ -264,7 +266,7 @@ var _ = Describe("FeatureStore Controller", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(1))) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git -c http.sslVerify=false clone")) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git checkout " + ref)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring(featureRepoPath)) @@ -294,7 +296,7 @@ var _ = Describe("FeatureStore Controller", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init -t spark")) }) @@ -1145,7 +1147,7 @@ var _ = Describe("FeatureStore Controller", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) // check client config cm := &corev1.ConfigMap{} diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index fe8f3ecbc15..f010986a563 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -725,6 +725,16 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", } podSpec.InitContainers = append(podSpec.InitContainers, container) + + if applied.Services.RunFeastApplyOnInit != nil && *applied.Services.RunFeastApplyOnInit { + applyContainer := corev1.Container{ + Name: "feast-apply", + Image: getFeatureServerImage(), + Command: []string{"feast", "apply"}, + WorkingDir: featureRepoDir, + } + podSpec.InitContainers = append(podSpec.InitContainers, applyContainer) + } } } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index e5299d79119..a0bde270133 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -173,7 +173,7 @@ var _ = Describe("TLS Config", func() { feastDeploy := feast.initFeastDeploy() err = feast.setDeployment(feastDeploy) Expect(err).ToNot(HaveOccurred()) - Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(4)) Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) Expect(feastDeploy.Spec.Template.Spec.Containers[1].Command).To(ContainElements(ContainSubstring("--key"))) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 9ce1ecd749a..04fcb9830a4 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -99,6 +99,9 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { applied.Services = &feastdevv1.FeatureStoreServices{} } services := applied.Services + if services.RunFeastApplyOnInit == nil { + services.RunFeastApplyOnInit = boolPtr(true) + } if services.Registry != nil { // if remote registry not set, proceed w/ local registry defaults From 265e9dc3be0ff9b6ae33a6db31baa21f1230674c Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sat, 14 Mar 2026 22:17:30 +0530 Subject: [PATCH 2/4] fix: Replicas is not required Signed-off-by: ntkathole --- infra/feast-operator/api/v1/featurestore_types.go | 2 +- .../config/crd/bases/feast.dev_featurestores.yaml | 2 -- infra/feast-operator/dist/install.yaml | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index dd12a94f7cd..9567a9de2b5 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -86,7 +86,7 @@ type FeatureStoreSpec struct { // Mutually exclusive with services.scaling.autoscaling. // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 - Replicas *int32 `json:"replicas"` + Replicas *int32 `json:"replicas,omitempty"` } // FeastProjectDir defines how to create the feast project directory. diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 2533ae930de..cdf50015e0f 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -5699,7 +5699,6 @@ spec: type: object required: - feastProject - - replicas type: object x-kubernetes-validations: - message: replicas > 1 and services.scaling.autoscaling are mutually @@ -11467,7 +11466,6 @@ spec: type: object required: - feastProject - - replicas type: object x-kubernetes-validations: - message: replicas > 1 and services.scaling.autoscaling are mutually diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 46b42928841..ce156a2dfa5 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -5707,7 +5707,6 @@ spec: type: object required: - feastProject - - replicas type: object x-kubernetes-validations: - message: replicas > 1 and services.scaling.autoscaling are mutually @@ -11475,7 +11474,6 @@ spec: type: object required: - feastProject - - replicas type: object x-kubernetes-validations: - message: replicas > 1 and services.scaling.autoscaling are mutually From 61c77a574a2234c1802b93aae1bb42d0ad3aa982 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sun, 15 Mar 2026 12:57:20 +0530 Subject: [PATCH 3/4] fix: Fixed race condition in _apply_object Signed-off-by: ntkathole --- .../controller/featurestore_controller_test.go | 6 ++++++ .../internal/controller/services/services.go | 12 ++++++++++++ sdk/python/feast/infra/registry/sql.py | 14 ++++++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 40a66c24ef5..c122abcc397 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -739,6 +739,12 @@ var _ = Describe("FeatureStore Controller", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Name).To(Equal("feast-apply")) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Env).To(ContainElements( + corev1.EnvVar{Name: testEnvVarName, Value: testEnvVarValue}, + )) + Expect(deploy.Spec.Template.Spec.InitContainers[1].EnvFrom).NotTo(BeEmpty()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) registryContainer := services.GetRegistryContainer(*deploy) Expect(registryContainer.Env).To(HaveLen(1)) diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index f010986a563..ffd30db5007 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -733,6 +733,18 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 Command: []string{"feast", "apply"}, WorkingDir: featureRepoDir, } + // feast apply needs DB/store connectivity, so inherit env/envFrom + // from all server container configs (registry, online, offline). + for _, feastType := range []FeastServiceType{RegistryFeastType, OnlineFeastType, OfflineFeastType} { + if serverConfigs := feast.getServerConfigs(feastType); serverConfigs != nil { + if serverConfigs.OptionalCtrConfigs.Env != nil { + applyContainer.Env = envOverride(applyContainer.Env, *serverConfigs.OptionalCtrConfigs.Env) + } + if serverConfigs.OptionalCtrConfigs.EnvFrom != nil { + applyContainer.EnvFrom = append(applyContainer.EnvFrom, *serverConfigs.OptionalCtrConfigs.EnvFrom...) + } + } + } podSpec.InitContainers = append(podSpec.InitContainers, applyContainer) } } diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 197ca02d57a..2332d0fb3d0 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -997,10 +997,16 @@ def _apply_object( "last_updated_timestamp": update_time, "project_id": project, } - insert_stmt = insert(table).values( - values, - ) - conn.execute(insert_stmt) + try: + with conn.begin_nested(): + conn.execute(insert(table).values(values)) + except IntegrityError: + logger.info( + "Object %s in project %s already created by another " + "process, skipping.", + name, + project, + ) if not isinstance(obj, Project): self.apply_project( From c9be7aaecc7c061e54ba30ef36d9850d202f83d3 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sun, 15 Mar 2026 20:02:25 +0530 Subject: [PATCH 4/4] fix: Improve error messaging Signed-off-by: ntkathole --- .../controller/featurestore_controller.go | 6 +- .../internal/controller/services/services.go | 72 ++++++++++- .../controller/services/services_test.go | 116 ++++++++++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 6f67852a601..aa3ad3b03e5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -196,11 +196,15 @@ func (r *FeatureStoreReconciler) deployFeast(ctx context.Context, cr *feastdevv1 } else { isDeployAvailable := services.IsDeploymentAvailable(deployment.Status.Conditions) if !isDeployAvailable { + msg := feastdevv1.DeploymentNotAvailableMessage + if podMsg := feast.GetPodContainerFailureMessage(deployment); podMsg != "" { + msg = msg + ": " + podMsg + } condition = metav1.Condition{ Type: feastdevv1.ReadyType, Status: metav1.ConditionUnknown, Reason: feastdevv1.DeploymentNotAvailableReason, - Message: feastdevv1.DeploymentNotAvailableMessage, + Message: msg, } result = errResult diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index ffd30db5007..446b5ab8ae0 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -733,8 +733,9 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 Command: []string{"feast", "apply"}, WorkingDir: featureRepoDir, } - // feast apply needs DB/store connectivity, so inherit env/envFrom - // from all server container configs (registry, online, offline). + // feast apply needs DB/store connectivity, so inherit env, envFrom + // and volume mounts from all server container configs. + seen := map[string]bool{} for _, feastType := range []FeastServiceType{RegistryFeastType, OnlineFeastType, OfflineFeastType} { if serverConfigs := feast.getServerConfigs(feastType); serverConfigs != nil { if serverConfigs.OptionalCtrConfigs.Env != nil { @@ -743,6 +744,12 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 if serverConfigs.OptionalCtrConfigs.EnvFrom != nil { applyContainer.EnvFrom = append(applyContainer.EnvFrom, *serverConfigs.OptionalCtrConfigs.EnvFrom...) } + for _, vm := range feast.getVolumeMounts(feastType) { + if !seen[vm.MountPath] { + applyContainer.VolumeMounts = append(applyContainer.VolumeMounts, vm) + seen[vm.MountPath] = true + } + } } } podSpec.InitContainers = append(podSpec.InitContainers, applyContainer) @@ -1427,6 +1434,67 @@ func IsDeploymentAvailable(conditions []appsv1.DeploymentCondition) bool { return false } +// GetPodContainerFailureMessage inspects pods belonging to the given deployment +// and returns a human-readable message describing the first init or regular +// container that is in a failing state. Returns empty string if no failure found. +func (feast *FeastServices) GetPodContainerFailureMessage(deploy appsv1.Deployment) string { + podList := corev1.PodList{} + labels := feast.getLabels() + if err := feast.Handler.Client.List(feast.Handler.Context, &podList, + client.InNamespace(deploy.Namespace), + client.MatchingLabels(labels), + ); err != nil { + return "" + } + for i := range podList.Items { + pod := &podList.Items[i] + if msg := initContainerFailureMessage(pod); msg != "" { + return msg + } + if msg := containerFailureMessage(pod); msg != "" { + return msg + } + } + return "" +} + +func initContainerFailureMessage(pod *corev1.Pod) string { + for _, cs := range pod.Status.InitContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" && cs.State.Waiting.Reason != "PodInitializing" { + return "Init container '" + cs.Name + "' waiting: " + cs.State.Waiting.Reason + + messageIfPresent(cs.State.Waiting.Message) + } + if cs.State.Terminated != nil && cs.State.Terminated.ExitCode != 0 { + return "Init container '" + cs.Name + "' failed with exit code " + + strconv.Itoa(int(cs.State.Terminated.ExitCode)) + + messageIfPresent(cs.State.Terminated.Message) + } + } + return "" +} + +func containerFailureMessage(pod *corev1.Pod) string { + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" && cs.State.Waiting.Reason != "ContainerCreating" { + return "Container '" + cs.Name + "' waiting: " + cs.State.Waiting.Reason + + messageIfPresent(cs.State.Waiting.Message) + } + if cs.State.Terminated != nil && cs.State.Terminated.ExitCode != 0 { + return "Container '" + cs.Name + "' failed with exit code " + + strconv.Itoa(int(cs.State.Terminated.ExitCode)) + + messageIfPresent(cs.State.Terminated.Message) + } + } + return "" +} + +func messageIfPresent(msg string) string { + if msg != "" { + return " - " + msg + } + return "" +} + // GetFeastRestServiceName returns the feast REST service object name based on service type func (feast *FeastServices) GetFeastRestServiceName(feastType FeastServiceType) string { return feast.GetFeastServiceName(feastType) + "-rest" diff --git a/infra/feast-operator/internal/controller/services/services_test.go b/infra/feast-operator/internal/controller/services/services_test.go index d53caa3e25f..131cdb850df 100644 --- a/infra/feast-operator/internal/controller/services/services_test.go +++ b/infra/feast-operator/internal/controller/services/services_test.go @@ -573,3 +573,119 @@ var _ = Describe("Registry Service", func() { }) }) }) + +var _ = Describe("Pod Container Failure Messages", func() { + It("should detect init container in CrashLoopBackOff", func() { + pod := &corev1.Pod{ + Status: corev1.PodStatus{ + InitContainerStatuses: []corev1.ContainerStatus{ + { + Name: "feast-init", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}, + }, + }, + { + Name: "feast-apply", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + Message: "back-off 5m0s restarting failed container", + }, + }, + }, + }, + }, + } + msg := initContainerFailureMessage(pod) + Expect(msg).To(ContainSubstring("feast-apply")) + Expect(msg).To(ContainSubstring("CrashLoopBackOff")) + Expect(msg).To(ContainSubstring("back-off 5m0s")) + }) + + It("should detect init container terminated with non-zero exit code", func() { + pod := &corev1.Pod{ + Status: corev1.PodStatus{ + InitContainerStatuses: []corev1.ContainerStatus{ + { + Name: "feast-apply", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 1, + Message: "feast apply failed", + }, + }, + }, + }, + }, + } + msg := initContainerFailureMessage(pod) + Expect(msg).To(ContainSubstring("feast-apply")) + Expect(msg).To(ContainSubstring("exit code 1")) + Expect(msg).To(ContainSubstring("feast apply failed")) + }) + + It("should return empty for init containers still initializing", func() { + pod := &corev1.Pod{ + Status: corev1.PodStatus{ + InitContainerStatuses: []corev1.ContainerStatus{ + { + Name: "feast-init", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "PodInitializing", + }, + }, + }, + }, + }, + } + Expect(initContainerFailureMessage(pod)).To(BeEmpty()) + }) + + It("should detect regular container failure", func() { + pod := &corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "registry", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "image not found", + }, + }, + }, + }, + }, + } + msg := containerFailureMessage(pod) + Expect(msg).To(ContainSubstring("registry")) + Expect(msg).To(ContainSubstring("ImagePullBackOff")) + }) + + It("should return empty for healthy pods", func() { + pod := &corev1.Pod{ + Status: corev1.PodStatus{ + InitContainerStatuses: []corev1.ContainerStatus{ + { + Name: "feast-init", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}, + }, + }, + }, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "registry", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{}, + }, + }, + }, + }, + } + Expect(initContainerFailureMessage(pod)).To(BeEmpty()) + Expect(containerFailureMessage(pod)).To(BeEmpty()) + }) +})