diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index 6b710064eec..32e4991c7a7 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -445,6 +445,15 @@ type FeatureStoreServices struct { // pod anti-affinity rule to prefer spreading pods across nodes. // +optional Affinity *corev1.Affinity `json:"affinity,omitempty"` + // Tolerations are applied to the FeatureStore deployment pods, allowing them to + // be scheduled onto nodes with matching taints. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // NodeSelector is a selector which must be true for the FeatureStore deployment + // pods to fit on a node. This selector must match a node's labels for the pod to + // be scheduled on that node. + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` // ResourceClaims defines which ResourceClaims must be allocated // and reserved before the Pod is allowed to start. The resources // will be made available to those containers which consume them diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 2719c1f0706..61223d49257 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -455,6 +455,20 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { *out = new(corev1.Affinity) (*in).DeepCopyInto(*out) } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.ResourceClaims != nil { in, out := &in.ResourceClaims, &out.ResourceClaims *out = make([]corev1.PodResourceClaim, len(*in)) diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 677f1b0737d..7d22e0cfb72 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -1761,6 +1761,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -4359,6 +4366,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. @@ -8101,6 +8142,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -10740,6 +10788,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 5f3e867db64..f362deb5c71 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -1761,6 +1761,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -4359,6 +4366,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. @@ -8101,6 +8142,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -10740,6 +10788,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 3a9f7b6e80e..f69133bcf6a 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -1769,6 +1769,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -4367,6 +4374,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. @@ -8109,6 +8150,13 @@ spec: description: InitImage overrides the image for init containers (feast-init, feast-apply). type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the FeatureStore deployment + pods to fit on a node. + type: object offlineStore: description: OfflineStore configures the offline store service properties: @@ -10748,6 +10796,40 @@ spec: type: string type: object type: object + tolerations: + description: |- + Tolerations are applied to the FeatureStore deployment pods, allowing them to + be scheduled onto nodes with matching... + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the... + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field... + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. + type: string + type: object + type: array topologySpreadConstraints: description: TopologySpreadConstraints defines how pods are spread across topology domains. diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index e2563c89e13..cbe2a91180c 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -289,6 +289,11 @@ Set to an empty array to disable auto-injection. | | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#affinity-v1-core)_ | Affinity defines the pod scheduling constraints for the FeatureStore deployment. When scaling is enabled and this is not set, the operator auto-injects a soft pod anti-affinity rule to prefer spreading pods across nodes. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#toleration-v1-core) array_ | Tolerations are applied to the FeatureStore deployment pods, allowing them to +be scheduled onto nodes with matching taints. | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is a selector which must be true for the FeatureStore deployment +pods to fit on a node. This selector must match a node's labels for the pod to +be scheduled on that node. | | `resourceClaims` _[PodResourceClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podresourceclaim-v1-core) array_ | ResourceClaims defines which ResourceClaims must be allocated and reserved before the Pod is allowed to start. The resources will be made available to those containers which consume them diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 6964938c093..540cf0f5586 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -447,6 +447,7 @@ func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { feast.mountEmptyDirVolumes(podSpec) feast.mountUserDefinedVolumes(podSpec) feast.applyNodeSelector(podSpec) + feast.applyTolerations(podSpec) feast.applyTopologySpread(podSpec) feast.applyAffinity(podSpec) feast.applyResourceClaims(podSpec) @@ -1031,8 +1032,18 @@ func (feast *FeastServices) getNodeSelectorForType(feastType FeastServiceType) * } func (feast *FeastServices) applyNodeSelector(podSpec *corev1.PodSpec) { - // Merge node selectors from all services + cr := feast.Handler.FeatureStore + services := cr.Status.Applied.Services + + // Start with the pod-level node selector configured on the FeatureStore + // services, then overlay per-service container config node selectors + // (per-service selectors win on key conflicts). mergedNodeSelector := make(map[string]string) + if services != nil && len(services.NodeSelector) > 0 { + for k, v := range services.NodeSelector { + mergedNodeSelector[k] = v + } + } // Check all service types for node selector configuration allServiceTypes := append(feastServerTypes, UIFeastType) @@ -1055,6 +1066,14 @@ func (feast *FeastServices) applyNodeSelector(podSpec *corev1.PodSpec) { podSpec.NodeSelector = finalNodeSelector } +func (feast *FeastServices) applyTolerations(podSpec *corev1.PodSpec) { + services := feast.Handler.FeatureStore.Status.Applied.Services + + if services != nil && services.Tolerations != nil { + podSpec.Tolerations = services.Tolerations + } +} + func (feast *FeastServices) applyTopologySpread(podSpec *corev1.PodSpec) { cr := feast.Handler.FeatureStore services := cr.Status.Applied.Services diff --git a/infra/feast-operator/internal/controller/services/services_test.go b/infra/feast-operator/internal/controller/services/services_test.go index da3590674f1..d0956148ae6 100644 --- a/infra/feast-operator/internal/controller/services/services_test.go +++ b/infra/feast-operator/internal/controller/services/services_test.go @@ -495,6 +495,56 @@ var _ = Describe("Registry Service", func() { Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) }) + It("should apply top-level NodeSelector to pod spec when configured", func() { + featureStore.Spec.Services.NodeSelector = map[string]string{ + kubernetesOsLabel: linuxOS, + nodeTypeLabel: computeNodeType, + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify NodeSelector is applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + expectedNodeSelector := map[string]string{ + kubernetesOsLabel: linuxOS, + nodeTypeLabel: computeNodeType, + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + + It("should let per-service NodeSelector override top-level NodeSelector on conflicting keys", func() { + featureStore.Spec.Services.NodeSelector = map[string]string{ + kubernetesOsLabel: linuxOS, + nodeTypeLabel: computeNodeType, + } + registryNodeSelector := map[string]string{ + nodeTypeLabel: "registry", + zoneLabel: "us-west-1a", + } + featureStore.Spec.Services.Registry.Local.Server.ContainerConfigs.OptionalCtrConfigs.NodeSelector = ®istryNodeSelector + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify merged NodeSelector is applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + expectedNodeSelector := map[string]string{ + kubernetesOsLabel: linuxOS, + nodeTypeLabel: "registry", + zoneLabel: "us-west-1a", + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + It("should enable metrics on the online service when configured", func() { featureStore.Spec.Services.OnlineStore = &feastdevv1.OnlineStore{ Server: &feastdevv1.ServerConfigs{Metrics: ptr.To(true)}, @@ -557,6 +607,45 @@ var _ = Describe("Registry Service", func() { }) }) + Describe("Tolerations Configuration", func() { + It("should apply Tolerations to pod spec when configured", func() { + tolerations := []corev1.Toleration{ + { + Key: "dedicated", + Operator: corev1.TolerationOpEqual, + Value: "feast", + Effect: corev1.TaintEffectNoSchedule, + }, + } + featureStore.Spec.Services.Tolerations = tolerations + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify Tolerations are applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + Expect(deployment.Spec.Template.Spec.Tolerations).To(Equal(tolerations)) + }) + + It("should leave Tolerations empty when not configured", func() { + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify no Tolerations are applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + Expect(deployment.Spec.Template.Spec.Tolerations).To(BeEmpty()) + }) + }) + Describe("WorkerConfigs Configuration", func() { It("should apply WorkerConfigs to the online store command", func() { // Set WorkerConfigs for online store