From 6656ec142d161529f2ceb21b06d7998963871df0 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sat, 21 Feb 2026 20:59:14 +0530 Subject: [PATCH 1/5] feat: Horizontal scaling support to the Feast operator Signed-off-by: ntkathole --- .secrets.baseline | 16 +- docs/how-to-guides/feast-on-kubernetes.md | 6 +- docs/how-to-guides/scaling-feast.md | 155 +- .../api/v1/featurestore_types.go | 45 + .../api/v1/zz_generated.deepcopy.go | 83 + .../crd/bases/feast.dev_featurestores.yaml | 1178 +++++ infra/feast-operator/config/rbac/role.yaml | 11 + .../samples/v1_featurestore_scaling_hpa.yaml | 76 + .../v1_featurestore_scaling_static.yaml | 60 + infra/feast-operator/dist/install.yaml | 4631 +++++++++++------ infra/feast-operator/docs/api/markdown/ref.md | 52 + .../controller/featurestore_controller.go | 3 + .../internal/controller/services/scaling.go | 180 + .../controller/services/scaling_test.go | 586 +++ .../internal/controller/services/services.go | 87 +- .../internal/controller/services/util.go | 48 + 16 files changed, 5457 insertions(+), 1760 deletions(-) create mode 100644 infra/feast-operator/config/samples/v1_featurestore_scaling_hpa.yaml create mode 100644 infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml create mode 100644 infra/feast-operator/internal/controller/services/scaling.go create mode 100644 infra/feast-operator/internal/controller/services/scaling_test.go diff --git a/.secrets.baseline b/.secrets.baseline index 9f4b51f7f02..f9889c73206 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -90,6 +90,10 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -930,7 +934,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 657 + "line_number": 692 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -939,21 +943,21 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 615 + "line_number": 658 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1123 + "line_number": 1206 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1128 + "line_number": 1211 } ], "infra/feast-operator/api/v1alpha1/featurestore_types.go": [ @@ -1152,7 +1156,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 164 + "line_number": 178 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ @@ -1535,5 +1539,5 @@ } ] }, - "generated_at": "2026-02-19T06:53:49Z" + "generated_at": "2026-02-21T16:33:24Z" } diff --git a/docs/how-to-guides/feast-on-kubernetes.md b/docs/how-to-guides/feast-on-kubernetes.md index 5504dbd671a..e87e3efa8b2 100644 --- a/docs/how-to-guides/feast-on-kubernetes.md +++ b/docs/how-to-guides/feast-on-kubernetes.md @@ -65,7 +65,9 @@ spec: > _More advanced FeatureStore CR examples can be found in the feast-operator [samples directory](../../infra/feast-operator/config/samples)._ {% hint style="success" %} -Important note: Scaling a Feature Store Deployment should only be done if the configured data store(s) will support it. +**Scaling:** The Feast Operator supports horizontal scaling via static replicas, HPA autoscaling, or external autoscalers like [KEDA](https://keda.sh). Scaling requires DB-backed persistence for all enabled services. -Please check the how-to guide for some specific recommendations on [how to scale Feast](./scaling-feast.md). +See the [Horizontal Scaling with the Feast Operator](./scaling-feast.md#horizontal-scaling-with-the-feast-operator) guide for configuration details, or check the general recommendations on [how to scale Feast](./scaling-feast.md). {% endhint %} + +> _Sample scaling CRs are available at [`v1_featurestore_scaling_static.yaml`](../../infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml) and [`v1_featurestore_scaling_hpa.yaml`](../../infra/feast-operator/config/samples/v1_featurestore_scaling_hpa.yaml)._ diff --git a/docs/how-to-guides/scaling-feast.md b/docs/how-to-guides/scaling-feast.md index d0bd6aef8a0..b1fc76db09a 100644 --- a/docs/how-to-guides/scaling-feast.md +++ b/docs/how-to-guides/scaling-feast.md @@ -23,4 +23,157 @@ However, this process does not scale for large data sets, since it's executed on Feast supports pluggable [Compute Engines](../getting-started/components/compute-engine.md), that allow the materialization process to be scaled up. Aside from the local process, Feast supports a [Lambda-based materialization engine](https://rtd.feast.dev/en/master/#alpha-lambda-based-engine), and a [Bytewax-based materialization engine](https://rtd.feast.dev/en/master/#bytewax-engine). -Users may also be able to build an engine to scale up materialization using existing infrastructure in their organizations. \ No newline at end of file +Users may also be able to build an engine to scale up materialization using existing infrastructure in their organizations. + +### Horizontal Scaling with the Feast Operator + +When running Feast on Kubernetes with the [Feast Operator](./feast-on-kubernetes.md), you can horizontally scale the FeatureStore deployment by adding a `scaling` field to the `services` section of the FeatureStore CR. + +**Prerequisites:** Horizontal scaling requires **DB-backed persistence** for all enabled services (online store, offline store, and registry). File-based persistence (SQLite, DuckDB, `registry.db`) is incompatible with multiple replicas because these backends do not support concurrent access from multiple pods. + +#### Static Replicas + +Set a fixed number of replicas: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-scaling +spec: + feastProject: my_project + services: + scaling: + replicas: 3 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +``` + +#### Autoscaling with HPA + +Configure a HorizontalPodAutoscaler to dynamically scale based on metrics: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-autoscaling +spec: + feastProject: my_project + services: + scaling: + autoscaling: + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + resources: + requests: + cpu: 200m + memory: 256Mi + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +``` + +{% hint style="info" %} +When autoscaling is configured, the operator automatically sets the deployment strategy to `RollingUpdate` (instead of the default `Recreate`) to ensure zero-downtime scaling. You can override this by explicitly setting `deploymentStrategy` in the CR. +{% endhint %} + +#### Validation Rules + +The operator enforces the following rules: +- `replicas` and `autoscaling` are **mutually exclusive** -- you cannot set both. +- Scaling with `replicas > 1` or any `autoscaling` config is **rejected** if any enabled service uses file-based persistence. +- S3 (`s3://`) and GCS (`gs://`) backed registry file persistence is allowed with scaling, since these object stores support concurrent readers. + +#### Using KEDA (Kubernetes Event-Driven Autoscaling) + +[KEDA](https://keda.sh) is also supported as an external autoscaler. Rather than using the built-in `autoscaling` field, you can create a KEDA `ScaledObject` that targets the Feast deployment directly. + +When using KEDA, do **not** set the `scaling.autoscaling` field -- KEDA manages its own HPA. The operator will preserve the replica count set by KEDA since it does not override externally managed replicas. + +There are a few things you must configure manually when using KEDA: + +1. **Set the deployment strategy to `RollingUpdate`** -- The operator defaults to `Recreate` when no `scaling` config is present, which causes downtime on scale events. Override it explicitly: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-keda +spec: + feastProject: my_project + services: + deploymentStrategy: + type: RollingUpdate + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +``` + +2. **Ensure DB-backed persistence** -- The operator's persistence validation only applies when the built-in `scaling` field is used. With KEDA, you are responsible for ensuring all enabled services use DB-backed persistence (not SQLite, DuckDB, or local `registry.db`). + +3. **Create a KEDA `ScaledObject`** targeting the Feast deployment: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: feast-scaledobject +spec: + scaleTargetRef: + name: feast-sample-keda # must match the Feast deployment name + minReplicaCount: 2 + maxReplicaCount: 10 + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus.monitoring.svc:9090 + metricName: http_requests_total + query: sum(rate(http_requests_total{service="feast"}[2m])) + threshold: "100" +``` + +{% hint style="warning" %} +KEDA-created HPAs are not owned by the Feast operator. The operator will not interfere with them, but it also will not clean them up if the FeatureStore CR is deleted. You must manage the KEDA `ScaledObject` lifecycle independently. +{% endhint %} + +For the full API reference, see the [FeatureStore CRD reference](../../infra/feast-operator/docs/api/markdown/ref.md). \ No newline at end of file diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index 977fc586110..f2a5767044d 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -18,6 +18,7 @@ package v1 import ( appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -301,6 +302,40 @@ type FeatureStoreServices struct { DisableInitContainers bool `json:"disableInitContainers,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). Volumes []corev1.Volume `json:"volumes,omitempty"` + // Scaling configures horizontal scaling for the FeatureStore deployment. + // Requires DB-based persistence for all enabled services when replicas > 1 or autoscaling is configured. + Scaling *ScalingConfig `json:"scaling,omitempty"` +} + +// ScalingConfig configures horizontal scaling for the FeatureStore deployment. +// +kubebuilder:validation:XValidation:rule="!has(self.replicas) || !has(self.autoscaling)",message="replicas and autoscaling are mutually exclusive." +type ScalingConfig struct { + // Replicas is the static number of pod replicas. Mutually exclusive with autoscaling. + // +kubebuilder:validation:Minimum=1 + // +optional + Replicas *int32 `json:"replicas,omitempty"` + // Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. + // Mutually exclusive with replicas. + // +optional + Autoscaling *AutoscalingConfig `json:"autoscaling,omitempty"` +} + +// AutoscalingConfig defines HPA settings for the FeatureStore deployment. +type AutoscalingConfig struct { + // MinReplicas is the lower limit for the number of replicas. Defaults to 1. + // +kubebuilder:validation:Minimum=1 + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + // MaxReplicas is the upper limit for the number of replicas. Required. + // +kubebuilder:validation:Minimum=1 + MaxReplicas int32 `json:"maxReplicas"` + // Metrics contains the specifications for which to use to calculate the desired replica count. + // If not set, defaults to 80% CPU utilization. + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + // Behavior configures the scaling behavior of the target. + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` } // OfflineStore configures the offline store service @@ -690,6 +725,16 @@ type FeatureStoreStatus struct { FeastVersion string `json:"feastVersion,omitempty"` Phase string `json:"phase,omitempty"` ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` + // ScalingStatus reports the current scaling state of the FeatureStore deployment. + ScalingStatus *ScalingStatus `json:"scalingStatus,omitempty"` +} + +// ScalingStatus reports the observed scaling state. +type ScalingStatus struct { + // CurrentReplicas is the current number of pod replicas. + CurrentReplicas int32 `json:"currentReplicas,omitempty"` + // DesiredReplicas is the desired number of pod replicas. + DesiredReplicas int32 `json:"desiredReplicas,omitempty"` } // ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 870f4489a4b..64756d89e82 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ package v1 import ( appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,6 +54,38 @@ func (in *AuthzConfig) DeepCopy() *AuthzConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalingConfig) DeepCopyInto(out *AutoscalingConfig) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingConfig. +func (in *AutoscalingConfig) DeepCopy() *AutoscalingConfig { + if in == nil { + return nil + } + out := new(AutoscalingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchEngineConfig) DeepCopyInto(out *BatchEngineConfig) { *out = *in @@ -342,6 +375,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Scaling != nil { + in, out := &in.Scaling, &out.Scaling + *out = new(ScalingConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreServices. @@ -406,6 +444,11 @@ func (in *FeatureStoreStatus) DeepCopyInto(out *FeatureStoreStatus) { } } out.ServiceHostnames = in.ServiceHostnames + if in.ScalingStatus != nil { + in, out := &in.ScalingStatus, &out.ScalingStatus + *out = new(ScalingStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreStatus. @@ -1044,6 +1087,46 @@ func (in *RemoteRegistryConfig) DeepCopy() *RemoteRegistryConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScalingConfig) DeepCopyInto(out *ScalingConfig) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = new(AutoscalingConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalingConfig. +func (in *ScalingConfig) DeepCopy() *ScalingConfig { + if in == nil { + return nil + } + out := new(ScalingConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScalingStatus) DeepCopyInto(out *ScalingStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalingStatus. +func (in *ScalingStatus) DeepCopy() *ScalingStatus { + if in == nil { + return nil + } + out := new(ScalingStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretKeyNames) DeepCopyInto(out *SecretKeyNames) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index a3acc201a1c..bdd294937b1 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -2353,6 +2353,587 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + scaling: + description: Scaling configures horizontal scaling for the FeatureStore + deployment. + properties: + autoscaling: + description: |- + Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. + Mutually exclusive with replicas. + properties: + behavior: + description: Behavior configures the scaling behavior + of the target. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + scaleUp: + description: scaleUp is scaling policy for scaling + Up. + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number + of replicas. Required. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics contains the specifications for which + to use to calculate the desired replica count. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at on + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes descr + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second) + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing eac + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: type is the type of metric source. + type: string + required: + - type + type: object + type: array + minReplicas: + description: MinReplicas is the lower limit for the number + of replicas. Defaults to 1. + format: int32 + minimum: 1 + type: integer + required: + - maxReplicas + type: object + replicas: + description: Replicas is the static number of pod replicas. + Mutually exclusive with autoscaling. + format: int32 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: replicas and autoscaling are mutually exclusive. + rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -6619,6 +7200,590 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + scaling: + description: Scaling configures horizontal scaling for the + FeatureStore deployment. + properties: + autoscaling: + description: |- + Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. + Mutually exclusive with replicas. + properties: + behavior: + description: Behavior configures the scaling behavior + of the target. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. + properties: + policies: + description: policies is a list of potential + scaling polices which can be used during + scaling. + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: periodSeconds specifies + the window of time for which the policy + should hold true. + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + scaleUp: + description: scaleUp is scaling policy for scaling + Up. + properties: + policies: + description: policies is a list of potential + scaling polices which can be used during + scaling. + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: periodSeconds specifies + the window of time for which the policy + should hold true. + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the + number of replicas. Required. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics contains the specifications for + which to use to calculate the desired replica count. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at on + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes descr + properties: + container: + description: container is the name of the + container in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. + properties: + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the + descriptions of a object,such as kind,name + apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the + referent; More info: https://git.k8s.' + type: string + name: + description: 'name is the name of the + referent; More info: https://kubernetes.' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second) + properties: + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing eac + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: type is the type of metric source. + type: string + required: + - type + type: object + type: array + minReplicas: + description: MinReplicas is the lower limit for the + number of replicas. Defaults to 1. + format: int32 + minimum: 1 + type: integer + required: + - maxReplicas + type: object + replicas: + description: Replicas is the static number of pod replicas. + Mutually exclusive with autoscaling. + format: int32 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: replicas and autoscaling are mutually exclusive. + rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -8604,6 +9769,19 @@ spec: type: string phase: type: string + scalingStatus: + description: ScalingStatus reports the current scaling state of the + FeatureStore deployment. + properties: + currentReplicas: + description: CurrentReplicas is the current number of pod replicas. + format: int32 + type: integer + desiredReplicas: + description: DesiredReplicas is the desired number of pod replicas. + format: int32 + type: integer + type: object serviceHostnames: description: ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index 56a952dd95c..ded745f1631 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -21,6 +21,17 @@ rules: - tokenreviews verbs: - create +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - batch resources: diff --git a/infra/feast-operator/config/samples/v1_featurestore_scaling_hpa.yaml b/infra/feast-operator/config/samples/v1_featurestore_scaling_hpa.yaml new file mode 100644 index 00000000000..4380b28d018 --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_scaling_hpa.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + namespace: test + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: feast + POSTGRES_PASSWORD: feast # pragma: allowlist secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores + namespace: test +stringData: + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.test.svc.cluster.local:5432/${POSTGRES_DB} + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true + postgres: | + host: postgres.test.svc.cluster.local + port: 5432 + database: ${POSTGRES_DB} + db_schema: public + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} +--- +# HPA autoscaling: 2-10 replicas with DB-backed persistence +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-scaling-hpa + namespace: test +spec: + feastProject: my_project + services: + scaling: + autoscaling: + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + envFrom: + - secretRef: + name: postgres-secret + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores diff --git a/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml b/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml new file mode 100644 index 00000000000..8eccdb7718a --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + namespace: test + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: feast + POSTGRES_PASSWORD: feast # pragma: allowlist secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores + namespace: test +stringData: + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.test.svc.cluster.local:5432/${POSTGRES_DB} + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true + postgres: | + host: postgres.test.svc.cluster.local + port: 5432 + database: ${POSTGRES_DB} + db_schema: public + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} +--- +# Static scaling: 3 replicas with DB-backed persistence +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-scaling-static + namespace: test +spec: + feastProject: my_project + services: + scaling: + replicas: 3 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + envFrom: + - secretRef: + name: postgres-secret + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index d42a902c886..bc111aaee18 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -2361,219 +2361,800 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' - securityContext: - description: PodSecurityContext holds pod-level security attributes - and common container settings. + scaling: + description: Scaling configures horizontal scaling for the FeatureStore + deployment. properties: - appArmorProfile: - description: appArmorProfile is the AppArmor options to use - by the containers in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile loaded - on the node that should be used. - type: string - type: - description: type indicates which kind of AppArmor profile - will be applied. - type: string - required: - - type - type: object - fsGroup: - description: A special supplemental group that applies to - all containers in a pod. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. - type: string - type: - description: type indicates which kind of seccomp profile - will be applied. - type: string - required: - - type - type: object - supplementalGroups: + autoscaling: description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsG - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: The Windows specific settings applied to all - containers. + Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. + Mutually exclusive with replicas. properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. - type: string - type: object - type: object - ui: - description: Creates a UI server container - properties: - env: - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. - type: string - value: + behavior: + description: Behavior configures the scaling behavior + of the target. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + scaleUp: + description: scaleUp is scaling policy for scaling + Up. + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number + of replicas. Required. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics contains the specifications for which + to use to calculate the desired replica count. + items: description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at on properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes descr properties: - key: - description: The key to select. + container: + description: container is the name of the container + in the pods of the scaling target type: string name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, `metadata.labels['''']`, - `metadata.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. + description: name is the name of the resource + in question. type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object required: - - fieldPath + - container + - name + - target type: object - x-kubernetes-map-type: atomic - resourceFieldRef: + external: description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits. + external refers to a global metric that is not associated + with any Kubernetes object. properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second) + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label selector + for the given metric\nWhen set, it is + passed " + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing eac + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: "averageUtilization is the + target value of the average of the\nresource + metric across all relevant pods, represented + as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: type is the type of metric source. + type: string + required: + - type + type: object + type: array + minReplicas: + description: MinReplicas is the lower limit for the number + of replicas. Defaults to 1. + format: int32 + minimum: 1 + type: integer + required: + - maxReplicas + type: object + replicas: + description: Replicas is the static number of pod replicas. + Mutually exclusive with autoscaling. + format: int32 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: replicas and autoscaling are mutually exclusive. + rule: '!has(self.replicas) || !has(self.autoscaling)' + securityContext: + description: PodSecurityContext holds pod-level security attributes + and common container settings. + properties: + appArmorProfile: + description: appArmorProfile is the AppArmor options to use + by the containers in this pod. + properties: + localhostProfile: + description: localhostProfile indicates a profile loaded + on the node that should be used. + type: string + type: + description: type indicates which kind of AppArmor profile + will be applied. + type: string + required: + - type + type: object + fsGroup: + description: A special supplemental group that applies to + all containers in a pod. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. + type: string + type: + description: type indicates which kind of seccomp profile + will be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in addition + to the container's primary GID, the fsG + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + description: Sysctls hold a list of namespaced sysctls used + for the pod. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: The Windows specific settings applied to all + containers. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. + type: string + type: object + type: object + ui: + description: Creates a UI server container + properties: + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource type: object x-kubernetes-map-type: atomic secretKeyRef: @@ -4302,65 +4883,460 @@ spec: name: default: "" description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + type: object + x-kubernetes-validations: + - message: One selection required between kubernetes or oidc. + rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, + c)' + batchEngine: + description: BatchEngineConfig defines the batch compute engine + configuration. + properties: + configMapKey: + description: Key name in the ConfigMap. Defaults to "config" + if not specified. + type: string + configMapRef: + description: Reference to a ConfigMap containing the batch + engine configuration. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + cronJob: + description: FeastCronJob defines a CronJob to execute against + a Feature Store deployment. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the CronJob metadata. + type: object + concurrencyPolicy: + description: Specifies how to treat concurrent executions + of a Job. + type: string + containerConfigs: + description: CronJobContainerConfigs k8s container settings + for the CronJob + properties: + commands: + description: Array of commands to be executed (in order) + against a Feature Store deployment. + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + description: PullPolicy describes a policy for if/when + to pull a container image + type: string + nodeSelector: + additionalProperties: + type: string + type: object + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes. + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum amount + of compute resources required. + type: object type: object - x-kubernetes-map-type: atomic - required: - - secretRef type: object - type: object - x-kubernetes-validations: - - message: One selection required between kubernetes or oidc. - rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, - c)' - batchEngine: - description: BatchEngineConfig defines the batch compute engine - configuration. - properties: - configMapKey: - description: Key name in the ConfigMap. Defaults to "config" - if not specified. - type: string - configMapRef: - description: Reference to a ConfigMap containing the batch - engine configuration. + failedJobsHistoryLimit: + description: The number of failed finished jobs to retain. + Value must be non-negative integer. + format: int32 + type: integer + jobSpec: + description: Specification of the desired behavior of a job. properties: - name: - default: "" + activeDeadlineSeconds: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tr + format: int64 + type: integer + backoffLimit: + description: Specifies the number of retries before marking + this job failed. + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. + format: int32 + type: integer + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. + format: int32 + type: integer + podFailurePolicy: + description: Specifies the policy of handling failed pods. + properties: + rules: + description: A list of pod failure policy rules. The + rules are evaluated in order. + items: + description: PodFailurePolicyRule describes how + a pod failure is handled when the requirements + are met. + properties: + action: + description: Specifies the action taken on a + pod failure when the requirements are satisfied. + type: string + onExitCodes: + description: Represents the requirement on the + container exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. + type: string + values: + description: Specifies the set of values. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: Specifies the required Pod + condition status. + type: string + type: + description: Specifies the required Pod + condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: podReplacementPolicy specifies when to create + replacement Pods. type: string + podTemplateAnnotations: + additionalProperties: + type: string + description: |- + PodTemplateAnnotations are annotations to be applied to the CronJob's PodTemplate + metadata. + type: object + suspend: + description: suspend specifies whether the Job controller + should create Pods or not. + type: boolean + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). + format: int32 + type: integer type: object - x-kubernetes-map-type: atomic + schedule: + description: The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. + type: string + startingDeadlineSeconds: + description: |- + Optional deadline in seconds for starting the job if it misses scheduled + time for any reason. + format: int64 + type: integer + successfulJobsHistoryLimit: + description: The number of successful finished jobs to retain. + Value must be non-negative integer. + format: int32 + type: integer + suspend: + description: |- + This flag tells the controller to suspend subsequent executions, it does + not apply to already started executions. + type: boolean + timeZone: + description: The time zone name for the given schedule, see + https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. + type: string type: object - cronJob: - description: FeastCronJob defines a CronJob to execute against - a Feature Store deployment. + feastProject: + description: FeastProject is the Feast project id. + pattern: ^[A-Za-z0-9][A-Za-z0-9_-]*$ + type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. properties: - annotations: - additionalProperties: - type: string - description: Annotations to be added to the CronJob metadata. - type: object - concurrencyPolicy: - description: Specifies how to treat concurrent executions - of a Job. - type: string - containerConfigs: - description: CronJobContainerConfigs k8s container settings - for the CronJob - properties: - commands: - description: Array of commands to be executed (in order) - against a Feature Store deployment. - items: + git: + description: GitCloneOptions describes how a clone should + be performed. + properties: + configs: + additionalProperties: type: string - type: array + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object env: items: description: EnvVar represents an environment variable @@ -4514,273 +5490,354 @@ spec: x-kubernetes-map-type: atomic type: object type: array - image: + featureRepoPath: + description: FeatureRepoPath is the relative path to the + feature repo subdirectory. Default is 'feature_repo'. type: string - imagePullPolicy: - description: PullPolicy describes a policy for if/when - to pull a container image + ref: + description: Reference to a branch / tag / commit type: string - nodeSelector: - additionalProperties: - type: string - type: object - resources: - description: ResourceRequirements describes the compute - resource requirements. + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast + init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + - clickhouse + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' + services: + description: FeatureStoreServices defines the desired feast services. + An ephemeral onlineStore feature server is deployed by default. + properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + maxSurge: + anyOf: + - type: integer + - type: string description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes. + The maximum number of pods that can be scheduled above the desired number of + pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: The maximum number of pods that can be + unavailable during the update. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean + offlineStore: + description: OfflineStore configures the offline store service + properties: + persistence: + description: OfflineStorePersistence configures the persistence + settings for the offline store service + properties: + file: + description: OfflineStoreFilePersistence configures + the file-based persistence for the offline store + service + properties: + pvc: + description: PvcConfig defines the settings for + a persistent file store based on PVCs. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: Resources describes the storage + resource requirements for a volume. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes. + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the + minimum amount of compute resources + required. + type: object + type: object + storageClassName: + description: StorageClassName is the name + of an existing StorageClass to which + this persistent volume belongs. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: + enum: + - file + - dask + - duckdb + type: string type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum amount - of compute resources required. + store: + description: OfflineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + type: + description: Type of the persistence type you + want to use. + enum: + - snowflake.offline + - bigquery + - redshift + - spark + - postgres + - trino + - athena + - mssql + - couchbase.offline + - clickhouse + - ray + type: string + required: + - secretRef + - type type: object type: object - type: object - failedJobsHistoryLimit: - description: The number of failed finished jobs to retain. - Value must be non-negative integer. - format: int32 - type: integer - jobSpec: - description: Specification of the desired behavior of a job. - properties: - activeDeadlineSeconds: - description: |- - Specifies the duration in seconds relative to the startTime that the job - may be continuously active before the system tr - format: int64 - type: integer - backoffLimit: - description: Specifies the number of retries before marking - this job failed. - format: int32 - type: integer - backoffLimitPerIndex: - description: |- - Specifies the limit for the number of retries within an - index before marking this index as failed. - format: int32 - type: integer - completionMode: - description: |- - completionMode specifies how Pod completions are tracked. It can be - `NonIndexed` (default) or `Indexed`. - type: string - completions: - description: |- - Specifies the desired number of successfully finished pods the - job should be run with. - format: int32 - type: integer - maxFailedIndexes: - description: |- - Specifies the maximal number of failed indexes before marking the Job as - failed, when backoffLimitPerIndex is set. - format: int32 - type: integer - parallelism: - description: |- - Specifies the maximum desired number of pods the job should - run at any given time. - format: int32 - type: integer - podFailurePolicy: - description: Specifies the policy of handling failed pods. + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' + server: + description: Creates a remote offline server container properties: - rules: - description: A list of pod failure policy rules. The - rules are evaluated in order. + env: items: - description: PodFailurePolicyRule describes how - a pod failure is handled when the requirements - are met. + description: EnvVar represents an environment variable + present in a Container. properties: - action: - description: Specifies the action taken on a - pod failure when the requirements are satisfied. + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string - onExitCodes: - description: Represents the requirement on the - container exit codes. - properties: - containerName: - description: |- - Restricts the check for exit codes to the container with the - specified name. - type: string - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. - type: string - values: - description: Specifies the set of values. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - - values - type: object - onPodConditions: + value: description: |- - Represents the requirement on the pod conditions. The requirement is represented - as a list of pod condition patterns. - items: - description: |- - PodFailurePolicyOnPodConditionsPattern describes a pattern for matching - an actual pod condition type. - properties: - status: - description: Specifies the required Pod - condition status. - type: string - type: - description: Specifies the required Pod - condition type. - type: string - required: - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - podReplacementPolicy: - description: podReplacementPolicy specifies when to create - replacement Pods. - type: string - podTemplateAnnotations: - additionalProperties: - type: string - description: |- - PodTemplateAnnotations are annotations to be applied to the CronJob's PodTemplate - metadata. - type: object - suspend: - description: suspend specifies whether the Job controller - should create Pods or not. - type: boolean - ttlSecondsAfterFinished: - description: |- - ttlSecondsAfterFinished limits the lifetime of a Job that has finished - execution (either Complete or Failed). - format: int32 - type: integer - type: object - schedule: - description: The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. - type: string - startingDeadlineSeconds: - description: |- - Optional deadline in seconds for starting the job if it misses scheduled - time for any reason. - format: int64 - type: integer - successfulJobsHistoryLimit: - description: The number of successful finished jobs to retain. - Value must be non-negative integer. - format: int32 - type: integer - suspend: - description: |- - This flag tells the controller to suspend subsequent executions, it does - not apply to already started executions. - type: boolean - timeZone: - description: The time zone name for the given schedule, see - https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. - type: string - type: object - feastProject: - description: FeastProject is the Feast project id. - pattern: ^[A-Za-z0-9][A-Za-z0-9_-]*$ - type: string - feastProjectDir: - description: FeastProjectDir defines how to create the feast project - directory. - properties: - git: - description: GitCloneOptions describes how a clone should - be performed. - properties: - configs: - additionalProperties: - type: string - description: |- - Configs passed to git via `-c` - e.g. http.sslVerify: 'false' - OR 'url."https://api:\${TOKEN}@github.com/". - type: object - env: - items: - description: EnvVar represents an environment variable - present in a Container. - properties: - name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any - type: string - valueFrom: - description: Source for the environment variable's - value. Cannot be used if value is not empty. + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. + configMapRef: + description: The ConfigMap to select from properties: - key: - description: The key to select. - type: string name: default: "" description: |- @@ -4790,62 +5847,17 @@ spec: type: string optional: description: Specify whether the ConfigMap - or its key must be defined + must be defined type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, `metadata.labels['''']`, - `metadata.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select - in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits. - properties: - containerName: - description: 'Container name: required for - volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource type: object x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the - pod's namespace + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from properties: - key: - description: The key of the secret to select - from. Must be a valid secret key. - type: string name: default: "" description: |- @@ -4855,153 +5867,228 @@ spec: type: string optional: description: Specify whether the Secret - or its key must be defined + must be defined type: boolean - required: - - key type: object x-kubernetes-map-type: atomic type: object - required: - - name - type: object - type: array - envFrom: - items: - description: EnvFromSource represents the source of - a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from + type: array + image: + type: string + imagePullPolicy: + description: PullPolicy describes a policy for if/when + to pull a container image + type: string + logLevel: + description: |- + LogLevel sets the logging level for the server + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + metrics: + description: Metrics exposes Prometheus-compatible + metrics for the Feast server when enabled. + type: boolean + nodeSelector: + additionalProperties: + type: string + type: object + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes. + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum amount + of compute resources required. + type: object + type: object + tls: + description: TlsConfigs configures server TLS for + a feast service. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, + where TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' + volumeMounts: + description: VolumeMounts defines the list of volumes + that should be mounted into the feast container. + items: + description: VolumeMount describes a mounting of + a Volume within a container. properties: - name: - default: "" + mountPath: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string - optional: - description: Specify whether the ConfigMap must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to - each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" + mountPropagation: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + mountPropagation determines how mounts are propagated from the host + to container and the other way around. type: string - optional: - description: Specify whether the Secret must - be defined + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should be + mounted. + type: string + required: + - mountPath + - name type: object - x-kubernetes-map-type: atomic - type: object - type: array - featureRepoPath: - description: FeatureRepoPath is the relative path to the - feature repo subdirectory. Default is 'feature_repo'. - type: string - ref: - description: Reference to a branch / tag / commit - type: string - url: - description: The repository URL to clone from. - type: string - required: - - url - type: object - x-kubernetes-validations: - - message: RepoPath must be a file name only, with no slashes. - rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') - : true' - init: - description: FeastInitOptions defines how to run a `feast - init`. - properties: - minimal: - type: boolean - template: - description: Template for the created project - enum: - - local - - gcp - - aws - - snowflake - - spark - - postgres - - hbase - - cassandra - - hazelcast - - ikv - - couchbase - - clickhouse - type: string - type: object - type: object - x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' - services: - description: FeatureStoreServices defines the desired feast services. - An ephemeral onlineStore feature server is deployed by default. - properties: - deploymentStrategy: - description: DeploymentStrategy describes how to replace existing - pods with new ones. - properties: - rollingUpdate: - description: |- - Rolling update config params. Present only if DeploymentStrategyType = - RollingUpdate. - properties: - maxSurge: - anyOf: - - type: integer - - type: string - description: |- - The maximum number of pods that can be scheduled above the desired number of - pods. - x-kubernetes-int-or-string: true - maxUnavailable: - anyOf: - - type: integer - - type: string - description: The maximum number of pods that can be - unavailable during the update. - x-kubernetes-int-or-string: true + type: array + workerConfigs: + description: WorkerConfigs defines the worker configuration + for the Feast server. + properties: + keepAliveTimeout: + description: |- + KeepAliveTimeout is the timeout for keep-alive connections in seconds. + Defaults to 30. + format: int32 + minimum: 1 + type: integer + maxRequests: + description: |- + MaxRequests is the maximum number of requests a worker will process before restarting. + This helps prevent memory leaks. + format: int32 + minimum: 0 + type: integer + maxRequestsJitter: + description: |- + MaxRequestsJitter is the maximum jitter to add to max-requests to prevent + thundering herd effect on worker restart. + format: int32 + minimum: 0 + type: integer + registryTTLSeconds: + description: RegistryTTLSeconds is the number + of seconds after which the registry is refreshed. + format: int32 + minimum: 0 + type: integer + workerConnections: + description: |- + WorkerConnections is the maximum number of simultaneous clients per worker process. + Defaults to 1000. + format: int32 + minimum: 1 + type: integer + workers: + description: Workers is the number of worker processes. + Use -1 to auto-calculate based on CPU cores + (2 * CPU + 1). + format: int32 + minimum: -1 + type: integer + type: object type: object - type: - description: Type of deployment. Can be "Recreate" or - "RollingUpdate". Default is RollingUpdate. - type: string type: object - disableInitContainers: - description: Disable the 'feast repo initialization' initContainer - type: boolean - offlineStore: - description: OfflineStore configures the offline store service + onlineStore: + description: OnlineStore configures the online store service properties: persistence: - description: OfflineStorePersistence configures the persistence - settings for the offline store service + description: OnlineStorePersistence configures the persistence + settings for the online store service properties: file: - description: OfflineStoreFilePersistence configures - the file-based persistence for the offline store + description: OnlineStoreFilePersistence configures + the file-based persistence for the online store service properties: + path: + type: string pvc: description: PvcConfig defines the settings for a persistent file store based on PVCs. @@ -5079,16 +6166,22 @@ spec: - message: Mount path must start with '/' and must not contain ':' rule: self.mountPath.matches('^/[^:]*$') - type: - enum: - - file - - dask - - duckdb - type: string type: object + x-kubernetes-validations: + - message: Ephemeral stores must have absolute paths. + rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') + : true' + - message: PVC path must be a file name only, with + no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: Online store does not support S3 or GS + buckets. + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: - description: OfflineStoreDBStorePersistence configures - the DB store persistence for the offline store service + description: OnlineStoreDBStorePersistence configures + the DB store persistence for the online store service properties: secretKeyName: description: By default, the selected store "type" @@ -5112,17 +6205,23 @@ spec: description: Type of the persistence type you want to use. enum: - - snowflake.offline - - bigquery - - redshift - - spark - - postgres - - trino - - athena - - mssql - - couchbase.offline - - clickhouse - - ray + - snowflake.online + - redis + - ikv + - datastore + - dynamodb + - bigtable + - postgres + - cassandra + - mysql + - hazelcast + - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase.online + - milvus + - hybrid type: string required: - secretRef @@ -5134,7 +6233,7 @@ spec: rule: '[has(self.file), has(self.store)].exists_one(c, c)' server: - description: Creates a remote offline server container + description: Creates a feature server container properties: env: items: @@ -5494,259 +6593,332 @@ spec: type: object type: object type: object - onlineStore: - description: OnlineStore configures the online store service + registry: + description: Registry configures the registry service. One + selection is required. Local is the default setting. properties: - persistence: - description: OnlineStorePersistence configures the persistence - settings for the online store service + local: + description: LocalRegistryConfig configures the registry + service properties: - file: - description: OnlineStoreFilePersistence configures - the file-based persistence for the online store - service + persistence: + description: RegistryPersistence configures the persistence + settings for the registry service properties: - path: - type: string - pvc: - description: PvcConfig defines the settings for - a persistent file store based on PVCs. + file: + description: RegistryFilePersistence configures + the file-based persistence for the registry + service properties: - create: - description: Settings for creating a new PVC + cache_mode: + description: |- + CacheMode defines the registry cache update strategy. + Allowed values are "sync" and "thread". + enum: + - none + - sync + - thread + type: string + cache_ttl_seconds: + description: CacheTTLSeconds defines the TTL + (in seconds) for the registry cache. + format: int32 + minimum: 0 + type: integer + path: + type: string + pvc: + description: PvcConfig defines the settings + for a persistent file store based on PVCs. properties: - accessModes: - description: AccessModes k8s persistent - volume access modes. Defaults to ["ReadWriteOnce"]. - items: - type: string - type: array - resources: - description: Resources describes the storage - resource requirements for a volume. + create: + description: Settings for creating a new + PVC properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes. - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the - minimum amount of compute resources - required. + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: Resources describes the + storage resource requirements for + a volume. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes. + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes + the minimum amount of compute + resources required. + type: object type: object + storageClassName: + description: StorageClassName is the + name of an existing StorageClass + to which this persistent volume + belongs. + type: string type: object - storageClassName: - description: StorageClassName is the name - of an existing StorageClass to which - this persistent volume belongs. + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. type: string + ref: + description: Reference to an existing + field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - - message: PvcCreate is immutable - rule: self == oldSelf - mountPath: - description: |- - MountPath within the container at which the volume should be mounted. - Must start by "/" and cannot contain ':'. - type: string - ref: - description: Reference to an existing field - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string + - message: One selection is required between + ref and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' + and must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + s3_additional_kwargs: + additionalProperties: + type: string type: object - x-kubernetes-map-type: atomic - required: - - mountPath type: object x-kubernetes-validations: - - message: One selection is required between ref - and create. - rule: '[has(self.ref), has(self.create)].exists_one(c, - c)' - - message: Mount path must start with '/' and - must not contain ':' - rule: self.mountPath.matches('^/[^:]*$') - type: object - x-kubernetes-validations: - - message: Ephemeral stores must have absolute paths. - rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') - : true' - - message: PVC path must be a file name only, with - no slashes. - rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') - : true' - - message: Online store does not support S3 or GS - buckets. - rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') - || self.path.startsWith(''gs://'')) : true' - store: - description: OnlineStoreDBStorePersistence configures - the DB store persistence for the online store service - properties: - secretKeyName: - description: By default, the selected store "type" - is used as the SecretKeyName - type: string - secretRef: - description: Data store parameters should be placed - as-is from the "feature_store.yaml" under the - secret key. + - message: Registry files must use absolute paths + or be S3 ('s3://') or GS ('gs://') object + store URIs. + rule: '(!has(self.pvc) && has(self.path)) ? + (self.path.startsWith(''/'') || self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: PVC path must be a file name only, + with no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: PVC persistence does not support S3 + or GS object store URIs. + rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: Additional S3 settings are available + only for S3 object store URIs. + rule: '(has(self.s3_additional_kwargs) && has(self.path)) + ? self.path.startsWith(''s3://'') : true' + store: + description: RegistryDBStorePersistence configures + the DB store persistence for the registry service properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + secretKeyName: + description: By default, the selected store + "type" is used as the SecretKeyName type: string + secretRef: + description: Data store parameters should + be placed as-is from the "feature_store.yaml" + under the secret key. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + type: + description: Type of the persistence type + you want to use. + enum: + - sql + - snowflake.registry + type: string + required: + - secretRef + - type type: object - x-kubernetes-map-type: atomic - type: - description: Type of the persistence type you - want to use. - enum: - - snowflake.online - - redis - - ikv - - datastore - - dynamodb - - bigtable - - postgres - - cassandra - - mysql - - hazelcast - - singlestore - - hbase - - elasticsearch - - qdrant - - couchbase.online - - milvus - - hybrid - type: string - required: - - secretRef - - type type: object - type: object - x-kubernetes-validations: - - message: One selection required between file or store. - rule: '[has(self.file), has(self.store)].exists_one(c, - c)' - server: - description: Creates a feature server container - properties: - env: - items: - description: EnvVar represents an environment variable - present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any - type: string - valueFrom: - description: Source for the environment variable's - value. Cannot be used if value is not empty. + x-kubernetes-validations: + - message: One selection required between file or + store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' + server: + description: Creates a registry server container + properties: + env: + items: + description: EnvVar represents an environment + variable present in a Container. properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: - key: - description: The key to select. - type: string - name: - default: "" + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - optional: - description: Specify whether the ConfigMap - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: - supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.' - properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, - defaults to "v1". - type: string - fieldPath: - description: Path of the field to select - in the specified API version. - type: string - required: - - fieldPath + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults - to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. type: string - required: - - resource + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean type: object x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in - the pod's namespace + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from properties: - key: - description: The key of the secret to - select from. Must be a valid secret - key. - type: string name: default: "" description: |- @@ -5756,154 +6928,262 @@ spec: type: string optional: description: Specify whether the Secret - or its key must be defined + must be defined type: boolean - required: - - key type: object x-kubernetes-map-type: atomic type: object - required: - - name - type: object - type: array - envFrom: - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from + type: array + grpc: + description: Enable gRPC registry server. Defaults + to true if unset. + type: boolean + image: + type: string + imagePullPolicy: + description: PullPolicy describes a policy for + if/when to pull a container image + type: string + logLevel: + description: |- + LogLevel sets the logging level for the server + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + metrics: + description: Metrics exposes Prometheus-compatible + metrics for the Feast server when enabled. + type: boolean + nodeSelector: + additionalProperties: + type: string + type: object + resources: + description: ResourceRequirements describes the + compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes. + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum + amount of compute resources required. + type: object + type: object + restAPI: + description: Enable REST API registry server. + type: boolean + tls: + description: TlsConfigs configures server TLS + for a feast service. + properties: + disable: + description: will disable TLS for the feast + service. useful in an openshift cluster, + for example, where TLS is configured by + default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret + where the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` + is false.' + rule: '(!has(self.disable) || !self.disable) + ? has(self.secretRef) : true' + volumeMounts: + description: VolumeMounts defines the list of + volumes that should be mounted into the feast + container. + items: + description: VolumeMount describes a mounting + of a Volume within a container. properties: - name: - default: "" + mountPath: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" + mountPropagation: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + mountPropagation determines how mounts are propagated from the host + to container and the other way around. type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - type: string - imagePullPolicy: - description: PullPolicy describes a policy for if/when - to pull a container image - type: string - logLevel: - description: |- - LogLevel sets the logging level for the server - Allowed values: "debug", "info", "warning", "error", "critical". - enum: - - debug - - info - - warning - - error - - critical - type: string - metrics: - description: Metrics exposes Prometheus-compatible - metrics for the Feast server when enabled. - type: boolean - nodeSelector: - additionalProperties: - type: string - type: object - resources: - description: ResourceRequirements describes the compute - resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + type: string + subPath: description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should + be mounted. type: string required: + - mountPath - name type: object type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes. - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum amount - of compute resources required. + workerConfigs: + description: WorkerConfigs defines the worker + configuration for the Feast server. + properties: + keepAliveTimeout: + description: |- + KeepAliveTimeout is the timeout for keep-alive connections in seconds. + Defaults to 30. + format: int32 + minimum: 1 + type: integer + maxRequests: + description: |- + MaxRequests is the maximum number of requests a worker will process before restarting. + This helps prevent memory leaks. + format: int32 + minimum: 0 + type: integer + maxRequestsJitter: + description: |- + MaxRequestsJitter is the maximum jitter to add to max-requests to prevent + thundering herd effect on worker restart. + format: int32 + minimum: 0 + type: integer + registryTTLSeconds: + description: RegistryTTLSeconds is the number + of seconds after which the registry is refreshed. + format: int32 + minimum: 0 + type: integer + workerConnections: + description: |- + WorkerConnections is the maximum number of simultaneous clients per worker process. + Defaults to 1000. + format: int32 + minimum: 1 + type: integer + workers: + description: Workers is the number of worker + processes. Use -1 to auto-calculate based + on CPU cores (2 * CPU + 1). + format: int32 + minimum: -1 + type: integer type: object type: object + x-kubernetes-validations: + - message: At least one of restAPI or grpc must be + true + rule: self.restAPI == true || self.grpc == true + || !has(self.grpc) + type: object + remote: + description: RemoteRegistryConfig points to a remote feast + registry server. + properties: + feastRef: + description: Reference to an existing `FeatureStore` + CR in the same k8s cluster. + properties: + name: + description: Name of the FeatureStore + type: string + namespace: + description: Namespace of the FeatureStore + type: string + required: + - name + type: object + hostname: + description: Host address of the remote registry service + - :, e.g. `registry..svc.cluster.local:80` + type: string tls: - description: TlsConfigs configures server TLS for - a feast service. + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. properties: - disable: - description: will disable TLS for the feast service. - useful in an openshift cluster, for example, - where TLS is configured by default - type: boolean - secretKeyNames: - description: SecretKeyNames defines the secret - key names for the TLS key and cert. - properties: - tlsCrt: - description: defaults to "tls.crt" - type: string - tlsKey: - description: defaults to "tls.key" - type: string - type: object - secretRef: - description: references the local k8s secret where - the TLS key and cert reside + certName: + description: defines the configmap key name for + the client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap + where the TLS cert resides properties: name: default: "" @@ -5914,719 +7194,604 @@ spec: type: string type: object x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: '`secretRef` required if `disable` is false.' - rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) - : true' - volumeMounts: - description: VolumeMounts defines the list of volumes - that should be mounted into the feast container. - items: - description: VolumeMount describes a mounting of - a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should be - mounted. - type: string - required: - - mountPath - - name - type: object - type: array - workerConfigs: - description: WorkerConfigs defines the worker configuration - for the Feast server. - properties: - keepAliveTimeout: - description: |- - KeepAliveTimeout is the timeout for keep-alive connections in seconds. - Defaults to 30. - format: int32 - minimum: 1 - type: integer - maxRequests: - description: |- - MaxRequests is the maximum number of requests a worker will process before restarting. - This helps prevent memory leaks. - format: int32 - minimum: 0 - type: integer - maxRequestsJitter: - description: |- - MaxRequestsJitter is the maximum jitter to add to max-requests to prevent - thundering herd effect on worker restart. - format: int32 - minimum: 0 - type: integer - registryTTLSeconds: - description: RegistryTTLSeconds is the number - of seconds after which the registry is refreshed. - format: int32 - minimum: 0 - type: integer - workerConnections: - description: |- - WorkerConnections is the maximum number of simultaneous clients per worker process. - Defaults to 1000. - format: int32 - minimum: 1 - type: integer - workers: - description: Workers is the number of worker processes. - Use -1 to auto-calculate based on CPU cores - (2 * CPU + 1). - format: int32 - minimum: -1 - type: integer + required: + - certName + - configMapRef type: object type: object + x-kubernetes-validations: + - message: One selection required. + rule: '[has(self.hostname), has(self.feastRef)].exists_one(c, + c)' type: object - registry: - description: Registry configures the registry service. One - selection is required. Local is the default setting. + x-kubernetes-validations: + - message: One selection required. + rule: '[has(self.local), has(self.remote)].exists_one(c, + c)' + scaling: + description: Scaling configures horizontal scaling for the + FeatureStore deployment. properties: - local: - description: LocalRegistryConfig configures the registry - service - properties: - persistence: - description: RegistryPersistence configures the persistence - settings for the registry service + autoscaling: + description: |- + Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. + Mutually exclusive with replicas. + properties: + behavior: + description: Behavior configures the scaling behavior + of the target. properties: - file: - description: RegistryFilePersistence configures - the file-based persistence for the registry - service + scaleDown: + description: scaleDown is scaling policy for scaling + Down. properties: - cache_mode: + policies: + description: policies is a list of potential + scaling polices which can be used during + scaling. + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: periodSeconds specifies + the window of time for which the policy + should hold true. + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: description: |- - CacheMode defines the registry cache update strategy. - Allowed values are "sync" and "thread". - enum: - - none - - sync - - thread + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. type: string - cache_ttl_seconds: - description: CacheTTLSeconds defines the TTL - (in seconds) for the registry cache. + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up format: int32 - minimum: 0 type: integer - path: - type: string - pvc: - description: PvcConfig defines the settings - for a persistent file store based on PVCs. - properties: - create: - description: Settings for creating a new - PVC - properties: - accessModes: - description: AccessModes k8s persistent - volume access modes. Defaults to - ["ReadWriteOnce"]. - items: - type: string - type: array - resources: - description: Resources describes the - storage resource requirements for - a volume. - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes. - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes - the minimum amount of compute - resources required. - type: object - type: object - storageClassName: - description: StorageClassName is the - name of an existing StorageClass - to which this persistent volume - belongs. - type: string - type: object - x-kubernetes-validations: - - message: PvcCreate is immutable - rule: self == oldSelf - mountPath: - description: |- - MountPath within the container at which the volume should be mounted. - Must start by "/" and cannot contain ':'. - type: string - ref: - description: Reference to an existing - field - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - mountPath - type: object - x-kubernetes-validations: - - message: One selection is required between - ref and create. - rule: '[has(self.ref), has(self.create)].exists_one(c, - c)' - - message: Mount path must start with '/' - and must not contain ':' - rule: self.mountPath.matches('^/[^:]*$') - s3_additional_kwargs: - additionalProperties: - type: string - type: object type: object - x-kubernetes-validations: - - message: Registry files must use absolute paths - or be S3 ('s3://') or GS ('gs://') object - store URIs. - rule: '(!has(self.pvc) && has(self.path)) ? - (self.path.startsWith(''/'') || self.path.startsWith(''s3://'') - || self.path.startsWith(''gs://'')) : true' - - message: PVC path must be a file name only, - with no slashes. - rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') - : true' - - message: PVC persistence does not support S3 - or GS object store URIs. - rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') - || self.path.startsWith(''gs://'')) : true' - - message: Additional S3 settings are available - only for S3 object store URIs. - rule: '(has(self.s3_additional_kwargs) && has(self.path)) - ? self.path.startsWith(''s3://'') : true' - store: - description: RegistryDBStorePersistence configures - the DB store persistence for the registry service + scaleUp: + description: scaleUp is scaling policy for scaling + Up. properties: - secretKeyName: - description: By default, the selected store - "type" is used as the SecretKeyName - type: string - secretRef: - description: Data store parameters should - be placed as-is from the "feature_store.yaml" - under the secret key. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - type: object - x-kubernetes-map-type: atomic - type: - description: Type of the persistence type - you want to use. - enum: - - sql - - snowflake.registry + policies: + description: policies is a list of potential + scaling polices which can be used during + scaling. + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: periodSeconds specifies + the window of time for which the policy + should hold true. + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. type: string - required: - - secretRef - - type + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up + format: int32 + type: integer type: object type: object - x-kubernetes-validations: - - message: One selection required between file or - store. - rule: '[has(self.file), has(self.store)].exists_one(c, - c)' - server: - description: Creates a registry server container - properties: - env: - items: - description: EnvVar represents an environment - variable present in a Container. + maxReplicas: + description: MaxReplicas is the upper limit for the + number of replicas. Required. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics contains the specifications for + which to use to calculate the desired replica count. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at on + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes descr properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + container: + description: container is the name of the + container in the pods of the scaling target type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any + name: + description: name is the name of the resource + in question. type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. + target: + description: target specifies the target + value for the given metric properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the - pod: supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits. + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. + properties: + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object type: object x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the + descriptions of a object,such as kind,name + apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the + referent; More info: https://git.k8s.' + type: string + name: + description: 'name is the name of the + referent; More info: https://kubernetes.' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object type: object x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type type: object required: - - name + - describedObject + - metric + - target type: object - type: array - envFrom: - items: - description: EnvFromSource represents the source - of a set of ConfigMaps + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second) properties: - configMapRef: - description: The ConfigMap to select from + metric: + description: metric identifies the target + metric by name and selector properties: name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + description: name is the name of the + given metric type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean + selector: + description: "selector is the string-encoded + form of a standard kubernetes label + selector for the given metric\nWhen + set, it is passed " + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from + target: + description: target specifies the target + value for the given metric properties: - name: - default: "" + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type type: object - x-kubernetes-map-type: atomic + required: + - metric + - target type: object - type: array - grpc: - description: Enable gRPC registry server. Defaults - to true if unset. - type: boolean - image: - type: string - imagePullPolicy: - description: PullPolicy describes a policy for - if/when to pull a container image - type: string - logLevel: - description: |- - LogLevel sets the logging level for the server - Allowed values: "debug", "info", "warning", "error", "critical". - enum: - - debug - - info - - warning - - error - - critical - type: string - metrics: - description: Metrics exposes Prometheus-compatible - metrics for the Feast server when enabled. - type: boolean - nodeSelector: - additionalProperties: - type: string - type: object - resources: - description: ResourceRequirements describes the - compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing eac + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target + value for the given metric properties: - name: + averageUtilization: + description: "averageUtilization is + the target value of the average of + the\nresource metric across all relevant + pods, represented as a " + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, Value, + or AverageValue type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true required: - - name + - type type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes. - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum - amount of compute resources required. - type: object - type: object - restAPI: - description: Enable REST API registry server. - type: boolean - tls: - description: TlsConfigs configures server TLS - for a feast service. - properties: - disable: - description: will disable TLS for the feast - service. useful in an openshift cluster, - for example, where TLS is configured by - default - type: boolean - secretKeyNames: - description: SecretKeyNames defines the secret - key names for the TLS key and cert. - properties: - tlsCrt: - description: defaults to "tls.crt" - type: string - tlsKey: - description: defaults to "tls.key" - type: string - type: object - secretRef: - description: references the local k8s secret - where the TLS key and cert reside - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - type: object - x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: '`secretRef` required if `disable` - is false.' - rule: '(!has(self.disable) || !self.disable) - ? has(self.secretRef) : true' - volumeMounts: - description: VolumeMounts defines the list of - volumes that should be mounted into the feast - container. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. - type: string required: - - mountPath - name + - target type: object - type: array - workerConfigs: - description: WorkerConfigs defines the worker - configuration for the Feast server. - properties: - keepAliveTimeout: - description: |- - KeepAliveTimeout is the timeout for keep-alive connections in seconds. - Defaults to 30. - format: int32 - minimum: 1 - type: integer - maxRequests: - description: |- - MaxRequests is the maximum number of requests a worker will process before restarting. - This helps prevent memory leaks. - format: int32 - minimum: 0 - type: integer - maxRequestsJitter: - description: |- - MaxRequestsJitter is the maximum jitter to add to max-requests to prevent - thundering herd effect on worker restart. - format: int32 - minimum: 0 - type: integer - registryTTLSeconds: - description: RegistryTTLSeconds is the number - of seconds after which the registry is refreshed. - format: int32 - minimum: 0 - type: integer - workerConnections: - description: |- - WorkerConnections is the maximum number of simultaneous clients per worker process. - Defaults to 1000. - format: int32 - minimum: 1 - type: integer - workers: - description: Workers is the number of worker - processes. Use -1 to auto-calculate based - on CPU cores (2 * CPU + 1). - format: int32 - minimum: -1 - type: integer - type: object - type: object - x-kubernetes-validations: - - message: At least one of restAPI or grpc must be - true - rule: self.restAPI == true || self.grpc == true - || !has(self.grpc) - type: object - remote: - description: RemoteRegistryConfig points to a remote feast - registry server. - properties: - feastRef: - description: Reference to an existing `FeatureStore` - CR in the same k8s cluster. - properties: - name: - description: Name of the FeatureStore - type: string - namespace: - description: Namespace of the FeatureStore - type: string - required: - - name - type: object - hostname: - description: Host address of the remote registry service - - :, e.g. `registry..svc.cluster.local:80` - type: string - tls: - description: TlsRemoteRegistryConfigs configures client - TLS for a remote feast registry. - properties: - certName: - description: defines the configmap key name for - the client TLS cert. - type: string - configMapRef: - description: references the local k8s configmap - where the TLS cert resides - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - certName - - configMapRef - type: object + type: + description: type is the type of metric source. + type: string + required: + - type + type: object + type: array + minReplicas: + description: MinReplicas is the lower limit for the + number of replicas. Defaults to 1. + format: int32 + minimum: 1 + type: integer + required: + - maxReplicas type: object - x-kubernetes-validations: - - message: One selection required. - rule: '[has(self.hostname), has(self.feastRef)].exists_one(c, - c)' + replicas: + description: Replicas is the static number of pod replicas. + Mutually exclusive with autoscaling. + format: int32 + minimum: 1 + type: integer type: object x-kubernetes-validations: - - message: One selection required. - rule: '[has(self.local), has(self.remote)].exists_one(c, - c)' + - message: replicas and autoscaling are mutually exclusive. + rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -8612,6 +9777,19 @@ spec: type: string phase: type: string + scalingStatus: + description: ScalingStatus reports the current scaling state of the + FeatureStore deployment. + properties: + currentReplicas: + description: CurrentReplicas is the current number of pod replicas. + format: int32 + type: integer + desiredReplicas: + description: DesiredReplicas is the desired number of pod replicas. + format: int32 + type: integer + type: object serviceHostnames: description: ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 @@ -17325,6 +18503,17 @@ rules: - tokenreviews verbs: - create +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - batch resources: diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index ce64e4dd3ec..e65bd6ceac0 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -28,6 +28,24 @@ _Appears in:_ | `oidc` _[OidcAuthz](#oidcauthz)_ | | +#### AutoscalingConfig + + + +AutoscalingConfig defines HPA settings for the FeatureStore deployment. + +_Appears in:_ +- [ScalingConfig](#scalingconfig) + +| Field | Description | +| --- | --- | +| `minReplicas` _integer_ | MinReplicas is the lower limit for the number of replicas. Defaults to 1. | +| `maxReplicas` _integer_ | MaxReplicas is the upper limit for the number of replicas. Required. | +| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#metricspec-v2-autoscaling) array_ | Metrics contains the specifications for which to use to calculate the desired replica count. +If not set, defaults to 80% CPU utilization. | +| `behavior` _[HorizontalPodAutoscalerBehavior](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#horizontalpodautoscalerbehavior-v2-autoscaling)_ | Behavior configures the scaling behavior of the target. | + + #### BatchEngineConfig @@ -223,6 +241,8 @@ _Appears in:_ | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podsecuritycontext-v1-core)_ | | | `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). | +| `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment. +Requires DB-based persistence for all enabled services when replicas > 1 or autoscaling is configured. | #### FeatureStoreSpec @@ -263,6 +283,7 @@ _Appears in:_ | `feastVersion` _string_ | | | `phase` _string_ | | | `serviceHostnames` _[ServiceHostnames](#servicehostnames)_ | | +| `scalingStatus` _[ScalingStatus](#scalingstatus)_ | ScalingStatus reports the current scaling state of the FeatureStore deployment. | #### GitCloneOptions @@ -747,6 +768,37 @@ _Appears in:_ | `tls` _[TlsRemoteRegistryConfigs](#tlsremoteregistryconfigs)_ | | +#### ScalingConfig + + + +ScalingConfig configures horizontal scaling for the FeatureStore deployment. + +_Appears in:_ +- [FeatureStoreServices](#featurestoreservices) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | Replicas is the static number of pod replicas. Mutually exclusive with autoscaling. | +| `autoscaling` _[AutoscalingConfig](#autoscalingconfig)_ | Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. +Mutually exclusive with replicas. | + + +#### ScalingStatus + + + +ScalingStatus reports the observed scaling state. + +_Appears in:_ +- [FeatureStoreStatus](#featurestorestatus) + +| Field | Description | +| --- | --- | +| `currentReplicas` _integer_ | CurrentReplicas is the current number of pod replicas. | +| `desiredReplicas` _integer_ | DesiredReplicas is the desired number of pod replicas. | + + #### SecretKeyNames diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index a9591d97c8a..b0f709cc157 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -22,6 +22,7 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -65,6 +66,7 @@ type FeatureStoreReconciler struct { // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -229,6 +231,7 @@ func (r *FeatureStoreReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.RoleBinding{}). Owns(&rbacv1.Role{}). Owns(&batchv1.CronJob{}). + Owns(&autoscalingv2.HorizontalPodAutoscaler{}). Watches(&feastdevv1.FeatureStore{}, handler.EnqueueRequestsFromMapFunc(r.mapFeastRefsToFeastRequests)) if services.IsOpenShift() { diff --git a/infra/feast-operator/internal/controller/services/scaling.go b/infra/feast-operator/internal/controller/services/scaling.go new file mode 100644 index 00000000000..c51b6357321 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/scaling.go @@ -0,0 +1,180 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "errors" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + defaultHPACPUUtilization int32 = 80 + defaultHPAMinReplicas int32 = 1 +) + +// validateScaling checks that scaling configuration is compatible with the +// persistence backends. File-based stores (SQLite, DuckDB, registry.db) +// cannot be safely shared across replicas. +func (feast *FeastServices) validateScaling() error { + cr := feast.Handler.FeatureStore + if !isScalingEnabled(cr) { + return nil + } + if isFilePersistence(cr) { + return errors.New( + "horizontal scaling (replicas > 1 or autoscaling) requires DB-backed persistence " + + "for all enabled services. File-based persistence (SQLite, DuckDB, registry.db) " + + "is incompatible with multiple replicas") + } + return nil +} + +// getDesiredReplicas returns the replica count the operator should set on the +// Deployment. When autoscaling is configured the Deployment replicas field is +// left to the HPA (nil is returned). Otherwise the static replica count is +// returned, defaulting to 1 when no scaling config is present. +func (feast *FeastServices) getDesiredReplicas() *int32 { + cr := feast.Handler.FeatureStore + if cr.Status.Applied.Services == nil || cr.Status.Applied.Services.Scaling == nil { + return nil + } + scaling := cr.Status.Applied.Services.Scaling + if scaling.Autoscaling != nil { + // HPA manages replicas; do not set them on the Deployment + return nil + } + if scaling.Replicas != nil { + r := *scaling.Replicas + return &r + } + return nil +} + +// createOrDeleteHPA reconciles the HorizontalPodAutoscaler for the FeatureStore +// deployment. If autoscaling is not configured, any existing HPA is deleted. +func (feast *FeastServices) createOrDeleteHPA() error { + cr := feast.Handler.FeatureStore + hpa := feast.initHPA() + + scaling := cr.Status.Applied.Services.Scaling + if scaling == nil || scaling.Autoscaling == nil { + return feast.Handler.DeleteOwnedFeastObj(hpa) + } + + logger := log.FromContext(feast.Handler.Context) + if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, hpa, controllerutil.MutateFn(func() error { + return feast.setHPA(hpa) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "HorizontalPodAutoscaler", hpa.Name, "operation", op) + } + + return nil +} + +func (feast *FeastServices) initHPA() *autoscalingv2.HorizontalPodAutoscaler { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: feast.GetObjectMeta(), + } + hpa.SetGroupVersionKind(autoscalingv2.SchemeGroupVersion.WithKind("HorizontalPodAutoscaler")) + return hpa +} + +func (feast *FeastServices) setHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) error { + cr := feast.Handler.FeatureStore + scaling := cr.Status.Applied.Services.Scaling + if scaling == nil || scaling.Autoscaling == nil { + return nil + } + autoscaling := scaling.Autoscaling + + hpa.Labels = feast.getLabels() + + deploy := feast.initFeastDeploy() + minReplicas := defaultHPAMinReplicas + if autoscaling.MinReplicas != nil { + minReplicas = *autoscaling.MinReplicas + } + + hpa.Spec = autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: deploy.Name, + }, + MinReplicas: &minReplicas, + MaxReplicas: autoscaling.MaxReplicas, + } + + if len(autoscaling.Metrics) > 0 { + hpa.Spec.Metrics = autoscaling.Metrics + } else { + hpa.Spec.Metrics = defaultHPAMetrics() + } + + if autoscaling.Behavior != nil { + hpa.Spec.Behavior = autoscaling.Behavior + } + + return controllerutil.SetControllerReference(cr, hpa, feast.Handler.Scheme) +} + +func defaultHPAMetrics() []autoscalingv2.MetricSpec { + utilization := defaultHPACPUUtilization + return []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: &utilization, + }, + }, + }, + } +} + +// updateScalingStatus updates the scaling status fields based on the current deployment state. +func (feast *FeastServices) updateScalingStatus() { + cr := feast.Handler.FeatureStore + if !isScalingEnabled(cr) { + cr.Status.ScalingStatus = nil + return + } + + deployment, err := feast.GetDeployment() + if err != nil { + return + } + + var desired int32 + if deployment.Spec.Replicas != nil { + desired = *deployment.Spec.Replicas + } + + cr.Status.ScalingStatus = &feastdevv1.ScalingStatus{ + CurrentReplicas: deployment.Status.ReadyReplicas, + DesiredReplicas: desired, + } +} diff --git a/infra/feast-operator/internal/controller/services/scaling_test.go b/infra/feast-operator/internal/controller/services/scaling_test.go new file mode 100644 index 00000000000..d1aa313533d --- /dev/null +++ b/infra/feast-operator/internal/controller/services/scaling_test.go @@ -0,0 +1,586 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "context" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("Horizontal Scaling", func() { + var ( + featureStore *feastdevv1.FeatureStore + feast *FeastServices + typeNamespacedName types.NamespacedName + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + typeNamespacedName = types.NamespacedName{ + Name: "scaling-test-fs", + Namespace: "default", + } + + featureStore = &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "scalingproject", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + }, + }, + Persistence: &feastdevv1.OnlineStorePersistence{ + DBPersistence: &feastdevv1.OnlineStoreDBStorePersistence{ + Type: "redis", + SecretRef: corev1.LocalObjectReference{ + Name: "redis-secret", + }, + }, + }, + }, + Registry: &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Server: &feastdevv1.RegistryServerConfigs{ + ServerConfigs: feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + }, + }, + GRPC: ptr(true), + }, + Persistence: &feastdevv1.RegistryPersistence{ + DBPersistence: &feastdevv1.RegistryDBStorePersistence{ + Type: "sql", + SecretRef: corev1.LocalObjectReference{ + Name: "registry-secret", + }, + }, + }, + }, + }, + }, + }, + } + + Expect(k8sClient.Create(ctx, featureStore)).To(Succeed()) + applySpecToStatus(featureStore) + + feast = &FeastServices{ + Handler: handler.FeastHandler{ + Client: k8sClient, + Context: ctx, + Scheme: k8sClient.Scheme(), + FeatureStore: featureStore, + }, + } + }) + + AfterEach(func() { + Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed()) + }) + + Describe("isScalingEnabled", func() { + It("should return false when no scaling config is present", func() { + Expect(isScalingEnabled(featureStore)).To(BeFalse()) + }) + + It("should return false when scaling config has replicas=1", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(1)), + } + Expect(isScalingEnabled(featureStore)).To(BeFalse()) + }) + + It("should return true when scaling config has replicas > 1", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + Expect(isScalingEnabled(featureStore)).To(BeTrue()) + }) + + It("should return true when autoscaling is configured", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + Expect(isScalingEnabled(featureStore)).To(BeTrue()) + }) + }) + + Describe("isFilePersistence", func() { + It("should return false when all services use DB persistence", func() { + Expect(isFilePersistence(featureStore)).To(BeFalse()) + }) + + It("should return true when online store uses file persistence", func() { + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + } + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return true when offline store uses file persistence", func() { + featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ + Persistence: &feastdevv1.OfflineStorePersistence{ + FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ + Type: "duckdb", + }, + }, + } + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return true when registry uses local file persistence", func() { + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "registry.db", + }, + } + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return false when registry uses S3 path", func() { + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "s3://my-bucket/registry.db", + }, + } + Expect(isFilePersistence(featureStore)).To(BeFalse()) + }) + + It("should return false when registry uses GS path", func() { + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "gs://my-bucket/registry.db", + }, + } + Expect(isFilePersistence(featureStore)).To(BeFalse()) + }) + }) + + Describe("validateScaling", func() { + It("should succeed when no scaling is configured", func() { + Expect(feast.validateScaling()).To(Succeed()) + }) + + It("should succeed with scaling and DB persistence", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + Expect(feast.validateScaling()).To(Succeed()) + }) + + It("should pass with scaling and S3-backed registry", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "s3://my-bucket/registry.db", + }, + } + Expect(feast.validateScaling()).To(Succeed()) + }) + + It("should pass with scaling and GS-backed registry", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "gs://my-bucket/registry.db", + }, + } + Expect(feast.validateScaling()).To(Succeed()) + }) + }) + + Describe("validateScaling rejects file-based persistence with scaling", func() { + It("should reject static replicas with file-based online store", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + Expect(err.Error()).To(ContainSubstring("File-based persistence")) + }) + + It("should reject autoscaling with file-based online store", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MinReplicas: ptr(int32(2)), + MaxReplicas: 10, + }, + } + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject static replicas with file-based online store using PVC", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(2)), + } + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "online.db", + PvcConfig: &feastdevv1.PvcConfig{ + Ref: &corev1.LocalObjectReference{Name: "my-pvc"}, + MountPath: "/data", + }, + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("File-based persistence")) + }) + + It("should reject static replicas with file-based offline store (duckdb)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ + Persistence: &feastdevv1.OfflineStorePersistence{ + FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ + Type: "duckdb", + }, + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject autoscaling with file-based offline store (dask)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ + Persistence: &feastdevv1.OfflineStorePersistence{ + FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ + Type: "dask", + }, + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject static replicas with file-based registry (registry.db)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "registry.db", + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject autoscaling with file-based registry", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "/data/registry.db", + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject scaling when only one service among many uses file persistence", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + // Online store uses DB persistence (from BeforeEach), but add file-based offline store + featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ + Persistence: &feastdevv1.OfflineStorePersistence{ + FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ + Type: "file", + }, + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should reject scaling when registry uses local file with PVC", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(2)), + } + featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "registry.db", + PvcConfig: &feastdevv1.PvcConfig{ + Ref: &corev1.LocalObjectReference{Name: "registry-pvc"}, + MountPath: "/data", + }, + }, + } + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("File-based persistence")) + }) + + It("should allow file persistence when replicas is 1 (no actual scaling)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(1)), + } + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + } + Expect(feast.validateScaling()).To(Succeed()) + }) + + It("should allow file persistence when scaling config is nil", func() { + featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + } + Expect(feast.validateScaling()).To(Succeed()) + }) + }) + + Describe("getDesiredReplicas", func() { + It("should return nil when no scaling is configured", func() { + Expect(feast.getDesiredReplicas()).To(BeNil()) + }) + + It("should return static replicas when configured", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + replicas := feast.getDesiredReplicas() + Expect(replicas).NotTo(BeNil()) + Expect(*replicas).To(Equal(int32(3))) + }) + + It("should return nil when autoscaling is configured (HPA manages replicas)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + Expect(feast.getDesiredReplicas()).To(BeNil()) + }) + }) + + Describe("Deployment Strategy", func() { + It("should default to Recreate when no scaling is configured", func() { + Expect(feast.ApplyDefaults()).To(Succeed()) + strategy := feast.getDeploymentStrategy() + Expect(strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) + }) + + It("should default to RollingUpdate when scaling is enabled", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + strategy := feast.getDeploymentStrategy() + Expect(strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) + }) + + It("should respect user-defined strategy even with scaling", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.DeploymentStrategy = &appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + } + strategy := feast.getDeploymentStrategy() + Expect(strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) + }) + }) + + Describe("setDeployment with scaling", func() { + // These tests use file-based (default) persistence to avoid needing real secrets + // for the feature_store.yaml generation that happens inside setDeployment. + setFilePersistence := func() { + featureStore.Status.Applied.Services.OnlineStore = &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + }, + }, + Persistence: &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/feast-data/online.db", + }, + }, + } + featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Server: &feastdevv1.RegistryServerConfigs{ + ServerConfigs: feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + }, + }, + GRPC: ptr(true), + }, + Persistence: &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "/feast-data/registry.db", + }, + }, + }, + } + } + + It("should set static replicas on the deployment", func() { + setFilePersistence() + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + deployment := feast.initFeastDeploy() + Expect(feast.setDeployment(deployment)).To(Succeed()) + Expect(deployment.Spec.Replicas).NotTo(BeNil()) + Expect(*deployment.Spec.Replicas).To(Equal(int32(3))) + }) + + It("should preserve existing replicas when autoscaling is configured", func() { + setFilePersistence() + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + deployment := feast.initFeastDeploy() + existing := int32(4) + deployment.Spec.Replicas = &existing + Expect(feast.setDeployment(deployment)).To(Succeed()) + Expect(deployment.Spec.Replicas).NotTo(BeNil()) + Expect(*deployment.Spec.Replicas).To(Equal(int32(4))) + }) + + It("should preserve existing replicas when no scaling is configured", func() { + setFilePersistence() + Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + deployment := feast.initFeastDeploy() + existing := int32(2) + deployment.Spec.Replicas = &existing + Expect(feast.setDeployment(deployment)).To(Succeed()) + Expect(deployment.Spec.Replicas).NotTo(BeNil()) + Expect(*deployment.Spec.Replicas).To(Equal(int32(2))) + }) + }) + + Describe("HPA Configuration", func() { + It("should create an HPA with default CPU metrics", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 10, + }, + } + + hpa := feast.initHPA() + Expect(feast.setHPA(hpa)).To(Succeed()) + Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) + Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1))) + Expect(hpa.Spec.Metrics).To(HaveLen(1)) + Expect(hpa.Spec.Metrics[0].Resource.Name).To(Equal(corev1.ResourceCPU)) + }) + + It("should create an HPA with custom min replicas", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MinReplicas: ptr(int32(2)), + MaxReplicas: 10, + }, + } + + hpa := feast.initHPA() + Expect(feast.setHPA(hpa)).To(Succeed()) + Expect(*hpa.Spec.MinReplicas).To(Equal(int32(2))) + Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) + }) + + It("should set correct scale target reference", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + + hpa := feast.initHPA() + Expect(feast.setHPA(hpa)).To(Succeed()) + Expect(hpa.Spec.ScaleTargetRef.APIVersion).To(Equal("apps/v1")) + Expect(hpa.Spec.ScaleTargetRef.Kind).To(Equal("Deployment")) + Expect(hpa.Spec.ScaleTargetRef.Name).To(Equal(GetFeastName(featureStore))) + }) + }) +}) diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 6771e9498af..3d5e2ddf37f 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -71,14 +71,47 @@ func (feast *FeastServices) Deploy() error { _ = feast.Handler.DeleteOwnedFeastObj(feast.initCaConfigMap()) } + if err := feast.reconcileServices(); err != nil { + return err + } + + if err := feast.validateScaling(); err != nil { + return err + } + if err := feast.createServiceAccount(); err != nil { + return err + } + if err := feast.createDeployment(); err != nil { + return err + } + if err := feast.createOrDeleteHPA(); err != nil { + return err + } + if err := feast.deployClient(); err != nil { + return err + } + if err := feast.deployNamespaceRegistry(); err != nil { + return err + } + if err := feast.deployCronJob(); err != nil { + return err + } + + feast.updateScalingStatus() + + return nil +} + +// reconcileServices validates persistence and deploys or removes each feast +// service type based on the applied spec. +func (feast *FeastServices) reconcileServices() error { services := feast.Handler.FeatureStore.Status.Applied.Services + if feast.isOfflineStore() { - err := feast.validateOfflineStorePersistence(services.OfflineStore.Persistence) - if err != nil { + if err := feast.validateOfflineStorePersistence(services.OfflineStore.Persistence); err != nil { return err } - - if err = feast.deployFeastServiceByType(OfflineFeastType); err != nil { + if err := feast.deployFeastServiceByType(OfflineFeastType); err != nil { return err } } else { @@ -88,12 +121,10 @@ func (feast *FeastServices) Deploy() error { } if feast.isOnlineStore() { - err := feast.validateOnlineStorePersistence(services.OnlineStore.Persistence) - if err != nil { + if err := feast.validateOnlineStorePersistence(services.OnlineStore.Persistence); err != nil { return err } - - if err = feast.deployFeastServiceByType(OnlineFeastType); err != nil { + if err := feast.deployFeastServiceByType(OnlineFeastType); err != nil { return err } } else { @@ -103,12 +134,10 @@ func (feast *FeastServices) Deploy() error { } if feast.isLocalRegistry() { - err := feast.validateRegistryPersistence(services.Registry.Local.Persistence) - if err != nil { + if err := feast.validateRegistryPersistence(services.Registry.Local.Persistence); err != nil { return err } - - if err = feast.deployFeastServiceByType(RegistryFeastType); err != nil { + if err := feast.deployFeastServiceByType(RegistryFeastType); err != nil { return err } } else { @@ -116,11 +145,12 @@ func (feast *FeastServices) Deploy() error { return err } } + if feast.isUiServer() { - if err = feast.deployFeastServiceByType(UIFeastType); err != nil { + if err := feast.deployFeastServiceByType(UIFeastType); err != nil { return err } - if err = feast.createRoute(UIFeastType); err != nil { + if err := feast.createRoute(UIFeastType); err != nil { return err } } else { @@ -132,22 +162,6 @@ func (feast *FeastServices) Deploy() error { } } - if err := feast.createServiceAccount(); err != nil { - return err - } - if err := feast.createDeployment(); err != nil { - return err - } - if err := feast.deployClient(); err != nil { - return err - } - if err := feast.deployNamespaceRegistry(); err != nil { - return err - } - if err := feast.deployCronJob(); err != nil { - return err - } - return nil } @@ -381,7 +395,15 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1.PvcCreate, feastType func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { cr := feast.Handler.FeatureStore + + // Determine replica count: + // - Static replicas from scaling config take precedence + // - When HPA is configured, preserve existing value so the HPA controller manages it + // - Otherwise preserve the existing value (supports external autoscalers) replicas := deploy.Spec.Replicas + if desired := feast.getDesiredReplicas(); desired != nil { + replicas = desired + } deploy.Labels = feast.getLabels() deploy.Spec = appsv1.DeploymentSpec{ @@ -635,6 +657,11 @@ func (feast *FeastServices) getDeploymentStrategy() appsv1.DeploymentStrategy { if feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy != nil { return *feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy } + if isScalingEnabled(feast.Handler.FeatureStore) { + return appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + } + } return appsv1.DeploymentStrategy{ Type: appsv1.RecreateDeploymentStrategyType, } diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 33d750251e9..47efbdfb063 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -493,6 +493,54 @@ func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1.F return nil } +// isScalingEnabled returns true when the user has configured horizontal scaling +// with either static replicas > 1 or HPA autoscaling. +func isScalingEnabled(featureStore *feastdevv1.FeatureStore) bool { + services := featureStore.Status.Applied.Services + if services == nil || services.Scaling == nil { + return false + } + scaling := services.Scaling + if scaling.Replicas != nil && *scaling.Replicas > 1 { + return true + } + return scaling.Autoscaling != nil +} + +// isFilePersistence returns true if any enabled service uses file-based persistence +// (SQLite, DuckDB, Dask, registry.db, etc.) rather than an external database. +func isFilePersistence(featureStore *feastdevv1.FeatureStore) bool { + services := featureStore.Status.Applied.Services + if services == nil { + return false + } + + if services.OnlineStore != nil && services.OnlineStore.Persistence != nil && + services.OnlineStore.Persistence.FilePersistence != nil { + return true + } + + if services.OfflineStore != nil && services.OfflineStore.Persistence != nil && + services.OfflineStore.Persistence.FilePersistence != nil { + return true + } + + if IsLocalRegistry(featureStore) && + services.Registry.Local.Persistence != nil && + services.Registry.Local.Persistence.FilePersistence != nil { + // S3/GS-backed registry file persistence is safe for multi-replica + if services.Registry.Local.Persistence.FilePersistence.Path != "" { + path := services.Registry.Local.Persistence.FilePersistence.Path + if strings.HasPrefix(path, "s3://") || strings.HasPrefix(path, "gs://") { + return false + } + } + return true + } + + return false +} + func boolPtr(value bool) *bool { return &value } From 9241ba992d89418c08392d9a838d2f3d0105e76b Mon Sep 17 00:00:00 2001 From: ntkathole Date: Mon, 23 Feb 2026 13:58:36 +0530 Subject: [PATCH 2/5] docs: Added blog post on horizontal scaling Signed-off-by: ntkathole --- .../controller/services/scaling_test.go | 52 ++++ .../internal/controller/services/util.go | 30 ++- .../docs/blog/scaling-feast-feature-server.md | 253 ++++++++++++++++++ 3 files changed, 326 insertions(+), 9 deletions(-) create mode 100644 infra/website/docs/blog/scaling-feast-feature-server.md diff --git a/infra/feast-operator/internal/controller/services/scaling_test.go b/infra/feast-operator/internal/controller/services/scaling_test.go index d1aa313533d..d00dba9db36 100644 --- a/infra/feast-operator/internal/controller/services/scaling_test.go +++ b/infra/feast-operator/internal/controller/services/scaling_test.go @@ -192,6 +192,36 @@ var _ = Describe("Horizontal Scaling", func() { } Expect(isFilePersistence(featureStore)).To(BeFalse()) }) + + It("should return true when no registry is configured (implicit file-based default)", func() { + featureStore.Status.Applied.Services.Registry = nil + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return true when local registry has no persistence configured", func() { + featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{}, + } + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return true when local registry has empty persistence", func() { + featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{}, + }, + } + Expect(isFilePersistence(featureStore)).To(BeTrue()) + }) + + It("should return false when remote registry is configured", func() { + featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ + Remote: &feastdevv1.RemoteRegistryConfig{ + Hostname: ptr("registry.example.com"), + }, + } + Expect(isFilePersistence(featureStore)).To(BeFalse()) + }) }) Describe("validateScaling", func() { @@ -229,6 +259,28 @@ var _ = Describe("Horizontal Scaling", func() { } Expect(feast.validateScaling()).To(Succeed()) }) + + It("should reject scaling when no registry is configured (implicit file-based default)", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.Registry = nil + err := feast.validateScaling() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + }) + + It("should succeed with scaling and remote registry", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Replicas: ptr(int32(3)), + } + featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ + Remote: &feastdevv1.RemoteRegistryConfig{ + Hostname: ptr("registry.example.com"), + }, + } + Expect(feast.validateScaling()).To(Succeed()) + }) }) Describe("validateScaling rejects file-based persistence with scaling", func() { diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 47efbdfb063..d3ee774a427 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -525,17 +525,29 @@ func isFilePersistence(featureStore *feastdevv1.FeatureStore) bool { return true } - if IsLocalRegistry(featureStore) && - services.Registry.Local.Persistence != nil && - services.Registry.Local.Persistence.FilePersistence != nil { - // S3/GS-backed registry file persistence is safe for multi-replica - if services.Registry.Local.Persistence.FilePersistence.Path != "" { - path := services.Registry.Local.Persistence.FilePersistence.Path - if strings.HasPrefix(path, "s3://") || strings.HasPrefix(path, "gs://") { - return false + // When no registry is configured, the deployment defaults to a file-based + // registry (registry.db). Only a remote registry or an explicit local + // registry with DB persistence is safe for multi-replica. + if services.Registry == nil { + return true + } + if isRemoteRegistry(featureStore) { + return false + } + if IsLocalRegistry(featureStore) { + if services.Registry.Local.Persistence == nil || + services.Registry.Local.Persistence.DBPersistence == nil { + // S3/GS-backed registry file persistence is safe for multi-replica + if services.Registry.Local.Persistence != nil && + services.Registry.Local.Persistence.FilePersistence != nil && + services.Registry.Local.Persistence.FilePersistence.Path != "" { + path := services.Registry.Local.Persistence.FilePersistence.Path + if strings.HasPrefix(path, "s3://") || strings.HasPrefix(path, "gs://") { + return false + } } + return true } - return true } return false diff --git a/infra/website/docs/blog/scaling-feast-feature-server.md b/infra/website/docs/blog/scaling-feast-feature-server.md new file mode 100644 index 00000000000..78fcf5154dd --- /dev/null +++ b/infra/website/docs/blog/scaling-feast-feature-server.md @@ -0,0 +1,253 @@ +--- +title: Scaling the Feast Feature Server on Kubernetes +description: The Feast Operator now supports horizontal scaling with static replicas, HPA autoscaling, and external autoscalers like KEDA — enabling production-grade, high-availability feature serving. +date: 2026-02-21 +authors: ["Nikhil Kathole"] +--- + +# Scaling the Feast Feature Server on Kubernetes + +As ML systems move from experimentation to production, the feature server often becomes a critical bottleneck. A single-replica deployment might handle development traffic, but production workloads — real-time inference, batch scoring, multiple consuming services — demand the ability to scale horizontally. + +We're excited to announce that the Feast Operator now supports **horizontal scaling** for the FeatureStore deployment, giving teams the tools to run Feast at production scale on Kubernetes. + +# The Problem: Single-Replica Limitations + +By default, the Feast Operator deploys a single-replica Deployment. This works well for getting started, but presents challenges as traffic grows: + +- **Single point of failure** — one pod crash means downtime for all feature consumers +- **Throughput ceiling** — a single pod can only handle so many concurrent requests +- **No elasticity** — traffic spikes (model retraining, batch inference) can overwhelm the server +- **Rolling updates cause downtime** — the default `Recreate` strategy tears down the old pod before starting a new one + +Teams have been manually patching Deployments or creating external HPAs, but this bypasses the operator's reconciliation loop and can lead to configuration drift. + +# The Solution: Native Scaling Support + +The Feast Operator now supports three scaling modes through a new `scaling` field in the FeatureStore CR: + +## 1. Static Replicas + +The simplest approach — set a fixed number of replicas: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: production-feast +spec: + feastProject: my_project + services: + scaling: + replicas: 3 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +``` + +This gives you high availability and load distribution with a predictable resource footprint. The operator automatically switches the Deployment strategy to `RollingUpdate`, ensuring zero-downtime deployments. + +## 2. HPA Autoscaling + +For workloads with variable traffic patterns, the operator can create and manage a `HorizontalPodAutoscaler` directly: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: autoscaled-feast +spec: + feastProject: my_project + services: + scaling: + autoscaling: + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +``` + +The operator creates the HPA as an owned resource — it's automatically cleaned up if you remove the autoscaling configuration or delete the FeatureStore CR. If no custom metrics are specified, the operator defaults to **80% CPU utilization**. + +## 3. External Autoscalers (KEDA, Custom HPAs) + +For teams using [KEDA](https://keda.sh) or other external autoscalers, the operator is designed to stay out of the way. When no `scaling` field is set, the operator preserves whatever replica count an external controller sets on the Deployment. + +To use KEDA, configure the FeatureStore with an explicit `RollingUpdate` strategy and DB-backed persistence, then create a KEDA `ScaledObject` targeting the Feast Deployment: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: keda-feast +spec: + feastProject: my_project + services: + deploymentStrategy: + type: RollingUpdate + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: feast-scaledobject +spec: + scaleTargetRef: + name: feast-keda-feast # matches the Feast deployment name + minReplicaCount: 2 + maxReplicaCount: 10 + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus.monitoring.svc:9090 + metricName: http_requests_total + query: sum(rate(http_requests_total{service="feast"}[2m])) + threshold: "100" +``` + +This gives you the full power of KEDA's 50+ event-driven triggers while the operator manages the rest of the Feast deployment lifecycle. + +# Safety First: Persistence Validation + +Not all persistence backends are safe for multi-replica deployments. File-based stores like SQLite, DuckDB, and local `registry.db` use single-writer file locks that don't work across pods. + +The operator enforces this at reconciliation time — if you configure scaling with file-based persistence, you'll get a clear error: + +``` +horizontal scaling (replicas > 1 or autoscaling) requires DB-backed persistence +for all enabled services. File-based persistence (SQLite, DuckDB, registry.db) +is incompatible with multiple replicas +``` + +This validation applies to all enabled services (online store, offline store, and registry). Object-store-backed registry paths (`s3://` and `gs://`) are treated as safe since they support concurrent readers. + +| Persistence Type | Compatible with Scaling? | +|---|---| +| PostgreSQL / MySQL | Yes | +| Redis | Yes | +| Cassandra | Yes | +| SQL-based Registry | Yes | +| S3/GCS Registry | Yes | +| SQLite | No | +| DuckDB | No | +| Local `registry.db` | No | + +# How It Works Under the Hood + +The implementation adds three key behaviors to the operator's reconciliation loop: + +**1. Replica management** — When static replicas are configured, the operator sets them on the Deployment. When HPA is configured, the operator leaves the `replicas` field unset so the HPA controller can manage it. When neither is configured, existing replicas are preserved (supporting external autoscalers). + +**2. Deployment strategy** — The operator automatically switches from `Recreate` (the default for single-replica) to `RollingUpdate` when scaling is enabled. This prevents the "kill-all-pods-then-start-new-ones" behavior that would cause downtime during scaling events. Users can always override this with an explicit `deploymentStrategy` in the CR. + +**3. HPA lifecycle** — The operator creates, updates, and deletes the HPA as an owned resource tied to the FeatureStore CR. Removing the `autoscaling` configuration automatically cleans up the HPA. + +The scaling status is reported back on the FeatureStore status: + +```yaml +status: + scalingStatus: + currentReplicas: 3 + desiredReplicas: 3 +``` + +# What About TLS, CronJobs, and Services? + +Scaling is designed to work seamlessly with existing operator features: + +- **TLS** — Each pod mounts the same TLS secret. OpenShift service-serving certificates work automatically since they're bound to the Service, not individual pods. +- **Kubernetes Services** — The Service's label selector already matches all pods in the Deployment, so load balancing across replicas works out of the box. +- **CronJobs** — The `feast apply` and `feast materialize-incremental` CronJobs use `kubectl exec` into a single pod. Since DB-backed persistence is required for scaling, all pods share the same state — it doesn't matter which pod the CronJob runs against. + +# Getting Started + +**1. Ensure DB-backed persistence** for all enabled services (online store, offline store, registry). + +**2. Add a `scaling` field** to your FeatureStore CR: + +```yaml +services: + scaling: + replicas: 3 # static replicas + # -- OR -- + # autoscaling: # HPA + # minReplicas: 2 + # maxReplicas: 10 +``` + +**3. Apply** the updated CR: + +```bash +kubectl apply -f my-featurestore.yaml +``` + +**4. Verify** the scaling: + +```bash +# Check pods +kubectl get pods -l app.kubernetes.io/managed-by=feast + +# Check HPA (if using autoscaling) +kubectl get hpa + +# Check FeatureStore status +kubectl get feast -o yaml +``` + +# Learn More + +- [Scaling Feast documentation](https://docs.feast.dev/how-to-guides/scaling-feast) +- [Feast on Kubernetes guide](https://docs.feast.dev/how-to-guides/feast-on-kubernetes) +- [FeatureStore CRD API reference](https://github.com/feast-dev/feast/blob/master/infra/feast-operator/docs/api/markdown/ref.md) +- [Sample CRs for static scaling and HPA](https://github.com/feast-dev/feast/tree/master/infra/feast-operator/config/samples) +- Join the [Feast Slack](https://slack.feast.dev) to share feedback and ask questions + +We're excited to see teams scale their feature serving infrastructure with confidence. Try it out and let us know how it works for your use case! From dee0cd6968b67ee6e8b15e93a6555caa6a617dec Mon Sep 17 00:00:00 2001 From: ntkathole Date: Thu, 26 Feb 2026 19:28:27 +0530 Subject: [PATCH 3/5] fix: Address comments Signed-off-by: ntkathole --- .secrets.baseline | 8 +- docs/how-to-guides/scaling-feast.md | 35 +- .../api/v1/featurestore_types.go | 25 +- .../api/v1/zz_generated.deepcopy.go | 10 +- .../crd/bases/feast.dev_featurestores.yaml | 117 +++- .../v1_featurestore_scaling_static.yaml | 3 +- infra/feast-operator/dist/install.yaml | 117 +++- infra/feast-operator/docs/api/markdown/ref.md | 11 +- .../featurestore_controller_test.go | 2 +- .../internal/controller/services/scaling.go | 57 +- .../controller/services/scaling_test.go | 622 ++++++++++-------- .../internal/controller/services/services.go | 12 +- .../internal/controller/services/util.go | 54 +- .../docs/blog/scaling-feast-feature-server.md | 51 +- 14 files changed, 627 insertions(+), 497 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index f9889c73206..26493e56777 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -934,7 +934,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 692 + "line_number": 696 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -943,7 +943,7 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 658 + "line_number": 663 }, { "type": "Secret Keyword", @@ -1156,7 +1156,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 178 + "line_number": 173 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ @@ -1539,5 +1539,5 @@ } ] }, - "generated_at": "2026-02-21T16:33:24Z" + "generated_at": "2026-02-26T14:08:35Z" } diff --git a/docs/how-to-guides/scaling-feast.md b/docs/how-to-guides/scaling-feast.md index b1fc76db09a..23c13bfe5ad 100644 --- a/docs/how-to-guides/scaling-feast.md +++ b/docs/how-to-guides/scaling-feast.md @@ -27,13 +27,17 @@ Users may also be able to build an engine to scale up materialization using exis ### Horizontal Scaling with the Feast Operator -When running Feast on Kubernetes with the [Feast Operator](./feast-on-kubernetes.md), you can horizontally scale the FeatureStore deployment by adding a `scaling` field to the `services` section of the FeatureStore CR. +When running Feast on Kubernetes with the [Feast Operator](./feast-on-kubernetes.md), you can horizontally scale the FeatureStore deployment using `spec.replicas` or HPA autoscaling. The FeatureStore CRD implements the Kubernetes [scale sub-resource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource), so you can also use `kubectl scale`: + +```bash +kubectl scale featurestore/my-feast --replicas=3 +``` **Prerequisites:** Horizontal scaling requires **DB-backed persistence** for all enabled services (online store, offline store, and registry). File-based persistence (SQLite, DuckDB, `registry.db`) is incompatible with multiple replicas because these backends do not support concurrent access from multiple pods. #### Static Replicas -Set a fixed number of replicas: +Set a fixed number of replicas via `spec.replicas`: ```yaml apiVersion: feast.dev/v1 @@ -42,9 +46,8 @@ metadata: name: sample-scaling spec: feastProject: my_project + replicas: 3 services: - scaling: - replicas: 3 onlineStore: persistence: store: @@ -62,7 +65,7 @@ spec: #### Autoscaling with HPA -Configure a HorizontalPodAutoscaler to dynamically scale based on metrics: +Configure a HorizontalPodAutoscaler to dynamically scale based on metrics. HPA autoscaling is configured under `services.scaling.autoscaling` and is mutually exclusive with `spec.replicas > 1`: ```yaml apiVersion: feast.dev/v1 @@ -110,19 +113,19 @@ When autoscaling is configured, the operator automatically sets the deployment s #### Validation Rules The operator enforces the following rules: -- `replicas` and `autoscaling` are **mutually exclusive** -- you cannot set both. +- `spec.replicas > 1` and `services.scaling.autoscaling` are **mutually exclusive** -- you cannot set both. - Scaling with `replicas > 1` or any `autoscaling` config is **rejected** if any enabled service uses file-based persistence. - S3 (`s3://`) and GCS (`gs://`) backed registry file persistence is allowed with scaling, since these object stores support concurrent readers. #### Using KEDA (Kubernetes Event-Driven Autoscaling) -[KEDA](https://keda.sh) is also supported as an external autoscaler. Rather than using the built-in `autoscaling` field, you can create a KEDA `ScaledObject` that targets the Feast deployment directly. +[KEDA](https://keda.sh) is also supported as an external autoscaler. KEDA should target the FeatureStore's scale sub-resource directly (since it implements the Kubernetes scale API). This is the recommended approach because the operator manages the Deployment's replica count from `spec.replicas` — targeting the Deployment directly would conflict with the operator's reconciliation. -When using KEDA, do **not** set the `scaling.autoscaling` field -- KEDA manages its own HPA. The operator will preserve the replica count set by KEDA since it does not override externally managed replicas. +When using KEDA, do **not** set `scaling.autoscaling` or `spec.replicas > 1` -- KEDA manages the replica count through the scale sub-resource. -There are a few things you must configure manually when using KEDA: +1. **Ensure DB-backed persistence** -- The CRD's CEL validation rules automatically enforce DB-backed persistence when KEDA scales `spec.replicas` above 1 via the scale sub-resource. The operator also automatically switches the deployment strategy to `RollingUpdate` when `replicas > 1`. -1. **Set the deployment strategy to `RollingUpdate`** -- The operator defaults to `Recreate` when no `scaling` config is present, which causes downtime on scale events. Override it explicitly: +2. **Configure the FeatureStore** with DB-backed persistence: ```yaml apiVersion: feast.dev/v1 @@ -132,8 +135,6 @@ metadata: spec: feastProject: my_project services: - deploymentStrategy: - type: RollingUpdate onlineStore: persistence: store: @@ -149,9 +150,7 @@ spec: name: feast-data-stores ``` -2. **Ensure DB-backed persistence** -- The operator's persistence validation only applies when the built-in `scaling` field is used. With KEDA, you are responsible for ensuring all enabled services use DB-backed persistence (not SQLite, DuckDB, or local `registry.db`). - -3. **Create a KEDA `ScaledObject`** targeting the Feast deployment: +3. **Create a KEDA `ScaledObject`** targeting the FeatureStore resource: ```yaml apiVersion: keda.sh/v1alpha1 @@ -160,8 +159,10 @@ metadata: name: feast-scaledobject spec: scaleTargetRef: - name: feast-sample-keda # must match the Feast deployment name - minReplicaCount: 2 + apiVersion: feast.dev/v1 + kind: FeatureStore + name: sample-keda + minReplicaCount: 1 maxReplicaCount: 10 triggers: - type: prometheus diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index f2a5767044d..8928fe74ce8 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -68,6 +68,10 @@ const ( ) // FeatureStoreSpec defines the desired state of FeatureStore +// +kubebuilder:validation:XValidation:rule="self.replicas <= 1 || !has(self.services) || !has(self.services.scaling) || !has(self.services.scaling.autoscaling)",message="replicas > 1 and services.scaling.autoscaling are mutually exclusive." +// +kubebuilder:validation:XValidation:rule="self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) || !has(self.services.scaling.autoscaling)) || (has(self.services) && has(self.services.onlineStore) && has(self.services.onlineStore.persistence) && has(self.services.onlineStore.persistence.store))",message="Scaling requires DB-backed persistence for the online store. Configure services.onlineStore.persistence.store when using replicas > 1 or autoscaling." +// +kubebuilder:validation:XValidation:rule="self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) || !has(self.services.scaling.autoscaling)) || (!has(self.services) || !has(self.services.offlineStore) || (has(self.services.offlineStore.persistence) && has(self.services.offlineStore.persistence.store)))",message="Scaling requires DB-backed persistence for the offline store. Configure services.offlineStore.persistence.store when using replicas > 1 or autoscaling." +// +kubebuilder:validation:XValidation:rule="self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) || !has(self.services.scaling.autoscaling)) || (has(self.services) && has(self.services.registry) && (has(self.services.registry.remote) || (has(self.services.registry.local) && has(self.services.registry.local.persistence) && (has(self.services.registry.local.persistence.store) || (has(self.services.registry.local.persistence.file) && has(self.services.registry.local.persistence.file.path) && (self.services.registry.local.persistence.file.path.startsWith('s3://') || self.services.registry.local.persistence.file.path.startsWith('gs://')))))))",message="Scaling requires DB-backed or remote registry. Configure registry.local.persistence.store or use a remote registry when using replicas > 1 or autoscaling. S3/GCS-backed registry is also allowed." type FeatureStoreSpec struct { // +kubebuilder:validation:Pattern="^[A-Za-z0-9][A-Za-z0-9_-]*$" // FeastProject is the Feast project id. This can be any alphanumeric string with underscores and hyphens, but it cannot start with an underscore or hyphen. Required. @@ -77,6 +81,11 @@ type FeatureStoreSpec struct { AuthzConfig *AuthzConfig `json:"authz,omitempty"` CronJob *FeastCronJob `json:"cronJob,omitempty"` BatchEngine *BatchEngineConfig `json:"batchEngine,omitempty"` + // Replicas is the desired number of pod replicas. Used by the scale sub-resource. + // Mutually exclusive with services.scaling.autoscaling. + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=1 + Replicas *int32 `json:"replicas"` } // FeastProjectDir defines how to create the feast project directory. @@ -302,20 +311,15 @@ type FeatureStoreServices struct { DisableInitContainers bool `json:"disableInitContainers,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). Volumes []corev1.Volume `json:"volumes,omitempty"` - // Scaling configures horizontal scaling for the FeatureStore deployment. - // Requires DB-based persistence for all enabled services when replicas > 1 or autoscaling is configured. + // Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). + // For static replicas, use spec.replicas instead. Scaling *ScalingConfig `json:"scaling,omitempty"` } // ScalingConfig configures horizontal scaling for the FeatureStore deployment. -// +kubebuilder:validation:XValidation:rule="!has(self.replicas) || !has(self.autoscaling)",message="replicas and autoscaling are mutually exclusive." type ScalingConfig struct { - // Replicas is the static number of pod replicas. Mutually exclusive with autoscaling. - // +kubebuilder:validation:Minimum=1 - // +optional - Replicas *int32 `json:"replicas,omitempty"` // Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. - // Mutually exclusive with replicas. + // Mutually exclusive with spec.replicas. // +optional Autoscaling *AutoscalingConfig `json:"autoscaling,omitempty"` } @@ -725,6 +729,10 @@ type FeatureStoreStatus struct { FeastVersion string `json:"feastVersion,omitempty"` Phase string `json:"phase,omitempty"` ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` + // Replicas is the current number of ready pod replicas (used by the scale sub-resource). + Replicas int32 `json:"replicas,omitempty"` + // Selector is the label selector for pods managed by the FeatureStore deployment (used by the scale sub-resource). + Selector string `json:"selector,omitempty"` // ScalingStatus reports the current scaling state of the FeatureStore deployment. ScalingStatus *ScalingStatus `json:"scalingStatus,omitempty"` } @@ -751,6 +759,7 @@ type ServiceHostnames struct { // +kubebuilder:resource:shortName=feast // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector // +kubebuilder:storageversion // FeatureStore is the Schema for the featurestores API diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 64756d89e82..6b12020435b 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -420,6 +420,11 @@ func (in *FeatureStoreSpec) DeepCopyInto(out *FeatureStoreSpec) { *out = new(BatchEngineConfig) (*in).DeepCopyInto(*out) } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreSpec. @@ -1090,11 +1095,6 @@ func (in *RemoteRegistryConfig) DeepCopy() *RemoteRegistryConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScalingConfig) DeepCopyInto(out *ScalingConfig) { *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } if in.Autoscaling != nil { in, out := &in.Autoscaling, &out.Autoscaling *out = new(AutoscalingConfig) diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index bdd294937b1..00a26ef5b58 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -712,6 +712,14 @@ spec: x-kubernetes-validations: - message: One selection required between init or git. rule: '[has(self.git), has(self.init)].exists_one(c, c)' + replicas: + default: 1 + description: |- + Replicas is the desired number of pod replicas. Used by the scale sub-resource. + Mutually exclusive with services. + format: int32 + minimum: 1 + type: integer services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -2355,12 +2363,12 @@ spec: rule: '[has(self.local), has(self.remote)].exists_one(c, c)' scaling: description: Scaling configures horizontal scaling for the FeatureStore - deployment. + deployment (e.g. HPA autoscaling). properties: autoscaling: description: |- Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. - Mutually exclusive with replicas. + Mutually exclusive with spec.replicas. properties: behavior: description: Behavior configures the scaling behavior @@ -2924,16 +2932,7 @@ spec: required: - maxReplicas type: object - replicas: - description: Replicas is the static number of pod replicas. - Mutually exclusive with autoscaling. - format: int32 - minimum: 1 - type: integer type: object - x-kubernetes-validations: - - message: replicas and autoscaling are mutually exclusive. - rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -4838,7 +4837,37 @@ spec: type: object required: - feastProject + - replicas type: object + x-kubernetes-validations: + - message: replicas > 1 and services.scaling.autoscaling are mutually + exclusive. + rule: self.replicas <= 1 || !has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling) + - message: Scaling requires DB-backed persistence for the online store. + Configure services.onlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.onlineStore) && has(self.services.onlineStore.persistence) + && has(self.services.onlineStore.persistence.store)) + - message: Scaling requires DB-backed persistence for the offline store. + Configure services.offlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (!has(self.services) + || !has(self.services.offlineStore) || (has(self.services.offlineStore.persistence) + && has(self.services.offlineStore.persistence.store))) + - message: Scaling requires DB-backed or remote registry. Configure registry.local.persistence.store + or use a remote registry when using replicas > 1 or autoscaling. S3/GCS-backed + registry is also allowed. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.registry) && (has(self.services.registry.remote) + || (has(self.services.registry.local) && has(self.services.registry.local.persistence) + && (has(self.services.registry.local.persistence.store) || (has(self.services.registry.local.persistence.file) + && has(self.services.registry.local.persistence.file.path) && (self.services.registry.local.persistence.file.path.startsWith('s3://') + || self.services.registry.local.persistence.file.path.startsWith('gs://'))))))) status: description: FeatureStoreStatus defines the observed state of FeatureStore properties: @@ -5526,6 +5555,14 @@ spec: x-kubernetes-validations: - message: One selection required between init or git. rule: '[has(self.git), has(self.init)].exists_one(c, c)' + replicas: + default: 1 + description: |- + Replicas is the desired number of pod replicas. Used by the scale sub-resource. + Mutually exclusive with services. + format: int32 + minimum: 1 + type: integer services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -7202,12 +7239,12 @@ spec: c)' scaling: description: Scaling configures horizontal scaling for the - FeatureStore deployment. + FeatureStore deployment (e.g. HPA autoscaling). properties: autoscaling: description: |- Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. - Mutually exclusive with replicas. + Mutually exclusive with spec.replicas. properties: behavior: description: Behavior configures the scaling behavior @@ -7774,16 +7811,7 @@ spec: required: - maxReplicas type: object - replicas: - description: Replicas is the static number of pod replicas. - Mutually exclusive with autoscaling. - format: int32 - minimum: 1 - type: integer type: object - x-kubernetes-validations: - - message: replicas and autoscaling are mutually exclusive. - rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -9704,7 +9732,39 @@ spec: type: object required: - feastProject + - replicas type: object + x-kubernetes-validations: + - message: replicas > 1 and services.scaling.autoscaling are mutually + exclusive. + rule: self.replicas <= 1 || !has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling) + - message: Scaling requires DB-backed persistence for the online store. + Configure services.onlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.onlineStore) && has(self.services.onlineStore.persistence) + && has(self.services.onlineStore.persistence.store)) + - message: Scaling requires DB-backed persistence for the offline + store. Configure services.offlineStore.persistence.store when + using replicas > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (!has(self.services) + || !has(self.services.offlineStore) || (has(self.services.offlineStore.persistence) + && has(self.services.offlineStore.persistence.store))) + - message: Scaling requires DB-backed or remote registry. Configure + registry.local.persistence.store or use a remote registry when + using replicas > 1 or autoscaling. S3/GCS-backed registry is also + allowed. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.registry) && (has(self.services.registry.remote) + || (has(self.services.registry.local) && has(self.services.registry.local.persistence) + && (has(self.services.registry.local.persistence.store) || (has(self.services.registry.local.persistence.file) + && has(self.services.registry.local.persistence.file.path) && + (self.services.registry.local.persistence.file.path.startsWith('s3://') + || self.services.registry.local.persistence.file.path.startsWith('gs://'))))))) clientConfigMap: description: ConfigMap in this namespace containing a client `feature_store.yaml` for this feast deployment @@ -9769,6 +9829,11 @@ spec: type: string phase: type: string + replicas: + description: Replicas is the current number of ready pod replicas + (used by the scale sub-resource). + format: int32 + type: integer scalingStatus: description: ScalingStatus reports the current scaling state of the FeatureStore deployment. @@ -9782,6 +9847,10 @@ spec: format: int32 type: integer type: object + selector: + description: Selector is the label selector for pods managed by the + FeatureStore deployment (used by the scale sub-resource). + type: string serviceHostnames: description: ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 @@ -9802,6 +9871,10 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas status: {} - additionalPrinterColumns: - jsonPath: .status.phase diff --git a/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml b/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml index 8eccdb7718a..c0f0f21cd6a 100644 --- a/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml +++ b/infra/feast-operator/config/samples/v1_featurestore_scaling_static.yaml @@ -38,9 +38,8 @@ metadata: namespace: test spec: feastProject: my_project + replicas: 3 services: - scaling: - replicas: 3 onlineStore: persistence: store: diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index bc111aaee18..38445ebfc1a 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -720,6 +720,14 @@ spec: x-kubernetes-validations: - message: One selection required between init or git. rule: '[has(self.git), has(self.init)].exists_one(c, c)' + replicas: + default: 1 + description: |- + Replicas is the desired number of pod replicas. Used by the scale sub-resource. + Mutually exclusive with services. + format: int32 + minimum: 1 + type: integer services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -2363,12 +2371,12 @@ spec: rule: '[has(self.local), has(self.remote)].exists_one(c, c)' scaling: description: Scaling configures horizontal scaling for the FeatureStore - deployment. + deployment (e.g. HPA autoscaling). properties: autoscaling: description: |- Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. - Mutually exclusive with replicas. + Mutually exclusive with spec.replicas. properties: behavior: description: Behavior configures the scaling behavior @@ -2932,16 +2940,7 @@ spec: required: - maxReplicas type: object - replicas: - description: Replicas is the static number of pod replicas. - Mutually exclusive with autoscaling. - format: int32 - minimum: 1 - type: integer type: object - x-kubernetes-validations: - - message: replicas and autoscaling are mutually exclusive. - rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -4846,7 +4845,37 @@ spec: type: object required: - feastProject + - replicas type: object + x-kubernetes-validations: + - message: replicas > 1 and services.scaling.autoscaling are mutually + exclusive. + rule: self.replicas <= 1 || !has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling) + - message: Scaling requires DB-backed persistence for the online store. + Configure services.onlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.onlineStore) && has(self.services.onlineStore.persistence) + && has(self.services.onlineStore.persistence.store)) + - message: Scaling requires DB-backed persistence for the offline store. + Configure services.offlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (!has(self.services) + || !has(self.services.offlineStore) || (has(self.services.offlineStore.persistence) + && has(self.services.offlineStore.persistence.store))) + - message: Scaling requires DB-backed or remote registry. Configure registry.local.persistence.store + or use a remote registry when using replicas > 1 or autoscaling. S3/GCS-backed + registry is also allowed. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.registry) && (has(self.services.registry.remote) + || (has(self.services.registry.local) && has(self.services.registry.local.persistence) + && (has(self.services.registry.local.persistence.store) || (has(self.services.registry.local.persistence.file) + && has(self.services.registry.local.persistence.file.path) && (self.services.registry.local.persistence.file.path.startsWith('s3://') + || self.services.registry.local.persistence.file.path.startsWith('gs://'))))))) status: description: FeatureStoreStatus defines the observed state of FeatureStore properties: @@ -5534,6 +5563,14 @@ spec: x-kubernetes-validations: - message: One selection required between init or git. rule: '[has(self.git), has(self.init)].exists_one(c, c)' + replicas: + default: 1 + description: |- + Replicas is the desired number of pod replicas. Used by the scale sub-resource. + Mutually exclusive with services. + format: int32 + minimum: 1 + type: integer services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -7210,12 +7247,12 @@ spec: c)' scaling: description: Scaling configures horizontal scaling for the - FeatureStore deployment. + FeatureStore deployment (e.g. HPA autoscaling). properties: autoscaling: description: |- Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. - Mutually exclusive with replicas. + Mutually exclusive with spec.replicas. properties: behavior: description: Behavior configures the scaling behavior @@ -7782,16 +7819,7 @@ spec: required: - maxReplicas type: object - replicas: - description: Replicas is the static number of pod replicas. - Mutually exclusive with autoscaling. - format: int32 - minimum: 1 - type: integer type: object - x-kubernetes-validations: - - message: replicas and autoscaling are mutually exclusive. - rule: '!has(self.replicas) || !has(self.autoscaling)' securityContext: description: PodSecurityContext holds pod-level security attributes and common container settings. @@ -9712,7 +9740,39 @@ spec: type: object required: - feastProject + - replicas type: object + x-kubernetes-validations: + - message: replicas > 1 and services.scaling.autoscaling are mutually + exclusive. + rule: self.replicas <= 1 || !has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling) + - message: Scaling requires DB-backed persistence for the online store. + Configure services.onlineStore.persistence.store when using replicas + > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.onlineStore) && has(self.services.onlineStore.persistence) + && has(self.services.onlineStore.persistence.store)) + - message: Scaling requires DB-backed persistence for the offline + store. Configure services.offlineStore.persistence.store when + using replicas > 1 or autoscaling. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (!has(self.services) + || !has(self.services.offlineStore) || (has(self.services.offlineStore.persistence) + && has(self.services.offlineStore.persistence.store))) + - message: Scaling requires DB-backed or remote registry. Configure + registry.local.persistence.store or use a remote registry when + using replicas > 1 or autoscaling. S3/GCS-backed registry is also + allowed. + rule: self.replicas <= 1 && (!has(self.services) || !has(self.services.scaling) + || !has(self.services.scaling.autoscaling)) || (has(self.services) + && has(self.services.registry) && (has(self.services.registry.remote) + || (has(self.services.registry.local) && has(self.services.registry.local.persistence) + && (has(self.services.registry.local.persistence.store) || (has(self.services.registry.local.persistence.file) + && has(self.services.registry.local.persistence.file.path) && + (self.services.registry.local.persistence.file.path.startsWith('s3://') + || self.services.registry.local.persistence.file.path.startsWith('gs://'))))))) clientConfigMap: description: ConfigMap in this namespace containing a client `feature_store.yaml` for this feast deployment @@ -9777,6 +9837,11 @@ spec: type: string phase: type: string + replicas: + description: Replicas is the current number of ready pod replicas + (used by the scale sub-resource). + format: int32 + type: integer scalingStatus: description: ScalingStatus reports the current scaling state of the FeatureStore deployment. @@ -9790,6 +9855,10 @@ spec: format: int32 type: integer type: object + selector: + description: Selector is the label selector for pods managed by the + FeatureStore deployment (used by the scale sub-resource). + type: string serviceHostnames: description: ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 @@ -9810,6 +9879,10 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas status: {} - additionalPrinterColumns: - jsonPath: .status.phase diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index e65bd6ceac0..1e2367a583f 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -241,8 +241,8 @@ _Appears in:_ | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podsecuritycontext-v1-core)_ | | | `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). | -| `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment. -Requires DB-based persistence for all enabled services when replicas > 1 or autoscaling is configured. | +| `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). +For static replicas, use spec.replicas instead. | #### FeatureStoreSpec @@ -263,6 +263,8 @@ _Appears in:_ | `authz` _[AuthzConfig](#authzconfig)_ | | | `cronJob` _[FeastCronJob](#feastcronjob)_ | | | `batchEngine` _[BatchEngineConfig](#batchengineconfig)_ | | +| `replicas` _integer_ | Replicas is the desired number of pod replicas. Used by the scale sub-resource. +Mutually exclusive with services.scaling.autoscaling. | #### FeatureStoreStatus @@ -283,6 +285,8 @@ _Appears in:_ | `feastVersion` _string_ | | | `phase` _string_ | | | `serviceHostnames` _[ServiceHostnames](#servicehostnames)_ | | +| `replicas` _integer_ | Replicas is the current number of ready pod replicas (used by the scale sub-resource). | +| `selector` _string_ | Selector is the label selector for pods managed by the FeatureStore deployment (used by the scale sub-resource). | | `scalingStatus` _[ScalingStatus](#scalingstatus)_ | ScalingStatus reports the current scaling state of the FeatureStore deployment. | @@ -779,9 +783,8 @@ _Appears in:_ | Field | Description | | --- | --- | -| `replicas` _integer_ | Replicas is the static number of pod replicas. Mutually exclusive with autoscaling. | | `autoscaling` _[AutoscalingConfig](#autoscalingconfig)_ | Autoscaling configures a HorizontalPodAutoscaler for the FeatureStore deployment. -Mutually exclusive with replicas. | +Mutually exclusive with spec.replicas. | #### ScalingStatus diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index bfd4a484cff..a70cd476679 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -263,7 +263,7 @@ var _ = Describe("FeatureStore Controller", func() { Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(3))) + Expect(deploy.Spec.Replicas).To(Equal(int32Ptr(1))) Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git -c http.sslVerify=false clone")) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git checkout " + ref)) diff --git a/infra/feast-operator/internal/controller/services/scaling.go b/infra/feast-operator/internal/controller/services/scaling.go index c51b6357321..7649a31d22b 100644 --- a/infra/feast-operator/internal/controller/services/scaling.go +++ b/infra/feast-operator/internal/controller/services/scaling.go @@ -17,11 +17,11 @@ limitations under the License. package services import ( - "errors" - feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -31,39 +31,18 @@ const ( defaultHPAMinReplicas int32 = 1 ) -// validateScaling checks that scaling configuration is compatible with the -// persistence backends. File-based stores (SQLite, DuckDB, registry.db) -// cannot be safely shared across replicas. -func (feast *FeastServices) validateScaling() error { - cr := feast.Handler.FeatureStore - if !isScalingEnabled(cr) { - return nil - } - if isFilePersistence(cr) { - return errors.New( - "horizontal scaling (replicas > 1 or autoscaling) requires DB-backed persistence " + - "for all enabled services. File-based persistence (SQLite, DuckDB, registry.db) " + - "is incompatible with multiple replicas") - } - return nil -} - // getDesiredReplicas returns the replica count the operator should set on the // Deployment. When autoscaling is configured the Deployment replicas field is -// left to the HPA (nil is returned). Otherwise the static replica count is -// returned, defaulting to 1 when no scaling config is present. +// left to the HPA (nil is returned). Otherwise the static replica count from +// spec.replicas is returned. func (feast *FeastServices) getDesiredReplicas() *int32 { cr := feast.Handler.FeatureStore - if cr.Status.Applied.Services == nil || cr.Status.Applied.Services.Scaling == nil { + services := cr.Status.Applied.Services + if services != nil && services.Scaling != nil && services.Scaling.Autoscaling != nil { return nil } - scaling := cr.Status.Applied.Services.Scaling - if scaling.Autoscaling != nil { - // HPA manages replicas; do not set them on the Deployment - return nil - } - if scaling.Replicas != nil { - r := *scaling.Replicas + if cr.Status.Applied.Replicas != nil { + r := *cr.Status.Applied.Replicas return &r } return nil @@ -155,26 +134,26 @@ func defaultHPAMetrics() []autoscalingv2.MetricSpec { } } -// updateScalingStatus updates the scaling status fields based on the current deployment state. -func (feast *FeastServices) updateScalingStatus() { +// updateScalingStatus updates the scaling status fields using the deployment +func (feast *FeastServices) updateScalingStatus(deploy *appsv1.Deployment) { cr := feast.Handler.FeatureStore + + cr.Status.Replicas = deploy.Status.ReadyReplicas + labels := feast.getLabels() + cr.Status.Selector = metav1.FormatLabelSelector(metav1.SetAsLabelSelector(labels)) + if !isScalingEnabled(cr) { cr.Status.ScalingStatus = nil return } - deployment, err := feast.GetDeployment() - if err != nil { - return - } - var desired int32 - if deployment.Spec.Replicas != nil { - desired = *deployment.Spec.Replicas + if deploy.Spec.Replicas != nil { + desired = *deploy.Spec.Replicas } cr.Status.ScalingStatus = &feastdevv1.ScalingStatus{ - CurrentReplicas: deployment.Status.ReadyReplicas, + CurrentReplicas: deploy.Status.ReadyReplicas, DesiredReplicas: desired, } } diff --git a/infra/feast-operator/internal/controller/services/scaling_test.go b/infra/feast-operator/internal/controller/services/scaling_test.go index d00dba9db36..dedb780f7e5 100644 --- a/infra/feast-operator/internal/controller/services/scaling_test.go +++ b/infra/feast-operator/internal/controller/services/scaling_test.go @@ -24,9 +24,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Horizontal Scaling", func() { @@ -117,17 +119,13 @@ var _ = Describe("Horizontal Scaling", func() { Expect(isScalingEnabled(featureStore)).To(BeFalse()) }) - It("should return false when scaling config has replicas=1", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(1)), - } + It("should return false when replicas=1", func() { + featureStore.Status.Applied.Replicas = ptr(int32(1)) Expect(isScalingEnabled(featureStore)).To(BeFalse()) }) - It("should return true when scaling config has replicas > 1", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } + It("should return true when replicas > 1", func() { + featureStore.Status.Applied.Replicas = ptr(int32(3)) Expect(isScalingEnabled(featureStore)).To(BeTrue()) }) @@ -141,329 +139,303 @@ var _ = Describe("Horizontal Scaling", func() { }) }) - Describe("isFilePersistence", func() { - It("should return false when all services use DB persistence", func() { - Expect(isFilePersistence(featureStore)).To(BeFalse()) - }) - - It("should return true when online store uses file persistence", func() { - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "/data/online.db", + Describe("CEL admission validation rejects invalid scaling configurations", func() { + dbOnlineStore := &feastdevv1.OnlineStore{ + Persistence: &feastdevv1.OnlineStorePersistence{ + DBPersistence: &feastdevv1.OnlineStoreDBStorePersistence{ + Type: "redis", + SecretRef: corev1.LocalObjectReference{Name: "redis-secret"}, }, - } - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) + }, + } - It("should return true when offline store uses file persistence", func() { - featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ - Persistence: &feastdevv1.OfflineStorePersistence{ - FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ - Type: "duckdb", + dbRegistry := &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{ + DBPersistence: &feastdevv1.RegistryDBStorePersistence{ + Type: "sql", + SecretRef: corev1.LocalObjectReference{Name: "registry-secret"}, }, }, - } - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) - - It("should return true when registry uses local file persistence", func() { - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "registry.db", - }, - } - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) - - It("should return false when registry uses S3 path", func() { - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "s3://my-bucket/registry.db", - }, - } - Expect(isFilePersistence(featureStore)).To(BeFalse()) - }) - - It("should return false when registry uses GS path", func() { - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "gs://my-bucket/registry.db", - }, - } - Expect(isFilePersistence(featureStore)).To(BeFalse()) - }) - - It("should return true when no registry is configured (implicit file-based default)", func() { - featureStore.Status.Applied.Services.Registry = nil - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) - - It("should return true when local registry has no persistence configured", func() { - featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ - Local: &feastdevv1.LocalRegistryConfig{}, - } - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) - - It("should return true when local registry has empty persistence", func() { - featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ - Local: &feastdevv1.LocalRegistryConfig{ - Persistence: &feastdevv1.RegistryPersistence{}, - }, - } - Expect(isFilePersistence(featureStore)).To(BeTrue()) - }) - - It("should return false when remote registry is configured", func() { - featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ - Remote: &feastdevv1.RemoteRegistryConfig{ - Hostname: ptr("registry.example.com"), - }, - } - Expect(isFilePersistence(featureStore)).To(BeFalse()) - }) - }) - - Describe("validateScaling", func() { - It("should succeed when no scaling is configured", func() { - Expect(feast.validateScaling()).To(Succeed()) - }) - - It("should succeed with scaling and DB persistence", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - Expect(feast.validateScaling()).To(Succeed()) - }) + }, + } - It("should pass with scaling and S3-backed registry", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "s3://my-bucket/registry.db", + It("should accept scaling with full DB persistence", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-valid-db", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: dbRegistry, + }, }, } - Expect(feast.validateScaling()).To(Succeed()) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should pass with scaling and GS-backed registry", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "gs://my-bucket/registry.db", + It("should reject scaling when online store is missing (implicit file default)", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-no-online", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + Registry: dbRegistry, + }, }, } - Expect(feast.validateScaling()).To(Succeed()) - }) - - It("should reject scaling when no registry is configured (implicit file-based default)", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.Registry = nil - err := feast.validateScaling() + err := k8sClient.Create(ctx, fs) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) - }) - - It("should succeed with scaling and remote registry", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.Registry = &feastdevv1.Registry{ - Remote: &feastdevv1.RemoteRegistryConfig{ - Hostname: ptr("registry.example.com"), + Expect(err.Error()).To(ContainSubstring("online store")) + }) + + It("should reject scaling when online store uses file persistence", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-file-online", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{ + Persistence: &feastdevv1.OnlineStorePersistence{ + FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ + Path: "/data/online.db", + }, + }, + }, + Registry: dbRegistry, + }, }, } - Expect(feast.validateScaling()).To(Succeed()) - }) - }) - - Describe("validateScaling rejects file-based persistence with scaling", func() { - It("should reject static replicas with file-based online store", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "/data/online.db", + err := k8sClient.Create(ctx, fs) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("online store")) + }) + + It("should reject scaling when offline store uses file persistence", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-file-offline", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: dbRegistry, + OfflineStore: &feastdevv1.OfflineStore{ + Persistence: &feastdevv1.OfflineStorePersistence{ + FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ + Type: "duckdb", + }, + }, + }, + }, }, } - err := feast.validateScaling() + err := k8sClient.Create(ctx, fs) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) - Expect(err.Error()).To(ContainSubstring("File-based persistence")) + Expect(err.Error()).To(ContainSubstring("offline store")) }) - It("should reject autoscaling with file-based online store", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Autoscaling: &feastdevv1.AutoscalingConfig{ - MinReplicas: ptr(int32(2)), - MaxReplicas: 10, + It("should reject scaling when no registry is configured (implicit file default)", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-no-registry", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + }, }, } - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "/data/online.db", + err := k8sClient.Create(ctx, fs) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("registry")) + }) + + It("should reject scaling when registry uses file persistence", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-file-registry", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "/data/registry.db", + }, + }, + }, + }, + }, }, } - err := feast.validateScaling() + err := k8sClient.Create(ctx, fs) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) - }) - - It("should reject static replicas with file-based online store using PVC", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(2)), - } - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "online.db", - PvcConfig: &feastdevv1.PvcConfig{ - Ref: &corev1.LocalObjectReference{Name: "my-pvc"}, - MountPath: "/data", + Expect(err.Error()).To(ContainSubstring("registry")) + }) + + It("should accept scaling with S3-backed registry", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-s3-registry", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "s3://my-bucket/registry.db", + }, + }, + }, + }, }, }, } - err := feast.validateScaling() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("File-based persistence")) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should reject static replicas with file-based offline store (duckdb)", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ - Persistence: &feastdevv1.OfflineStorePersistence{ - FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ - Type: "duckdb", + It("should accept scaling with GS-backed registry", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-gs-registry", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{ + FilePersistence: &feastdevv1.RegistryFilePersistence{ + Path: "gs://my-bucket/registry.db", + }, + }, + }, + }, }, }, } - err := feast.validateScaling() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should reject autoscaling with file-based offline store (dask)", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Autoscaling: &feastdevv1.AutoscalingConfig{ - MaxReplicas: 5, - }, - } - featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ - Persistence: &feastdevv1.OfflineStorePersistence{ - FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ - Type: "dask", + It("should accept scaling with remote registry", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-remote-reg", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: dbOnlineStore, + Registry: &feastdevv1.Registry{ + Remote: &feastdevv1.RemoteRegistryConfig{ + Hostname: ptr("registry.example.com:80"), + }, + }, }, }, } - err := feast.validateScaling() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should reject static replicas with file-based registry (registry.db)", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "registry.db", + It("should accept file persistence when replicas is 1", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-rep1-file", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(1)), }, } - err := feast.validateScaling() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should reject autoscaling with file-based registry", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Autoscaling: &feastdevv1.AutoscalingConfig{ - MaxReplicas: 5, - }, - } - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "/data/registry.db", + It("should accept file persistence when no scaling is configured", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-no-scaling", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", }, } - err := feast.validateScaling() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }) - It("should reject scaling when only one service among many uses file persistence", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - // Online store uses DB persistence (from BeforeEach), but add file-based offline store - featureStore.Status.Applied.Services.OfflineStore = &feastdevv1.OfflineStore{ - Persistence: &feastdevv1.OfflineStorePersistence{ - FilePersistence: &feastdevv1.OfflineStoreFilePersistence{ - Type: "file", + It("should reject autoscaling without DB online store", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-hpa-no-db", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Services: &feastdevv1.FeatureStoreServices{ + Scaling: &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{MaxReplicas: 5}, + }, + Registry: dbRegistry, }, }, } - err := feast.validateScaling() + err := k8sClient.Create(ctx, fs) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("DB-backed persistence")) - }) - - It("should reject scaling when registry uses local file with PVC", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(2)), - } - featureStore.Status.Applied.Services.Registry.Local.Persistence = &feastdevv1.RegistryPersistence{ - FilePersistence: &feastdevv1.RegistryFilePersistence{ - Path: "registry.db", - PvcConfig: &feastdevv1.PvcConfig{ - Ref: &corev1.LocalObjectReference{Name: "registry-pvc"}, - MountPath: "/data", + Expect(err.Error()).To(ContainSubstring("online store")) + }) + + It("should reject scaling when online store has no persistence configured", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-online-nop", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{}, + Registry: dbRegistry, }, }, } - err := feast.validateScaling() + err := k8sClient.Create(ctx, fs) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("File-based persistence")) - }) - - It("should allow file persistence when replicas is 1 (no actual scaling)", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(1)), - } - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "/data/online.db", - }, - } - Expect(feast.validateScaling()).To(Succeed()) - }) - - It("should allow file persistence when scaling config is nil", func() { - featureStore.Status.Applied.Services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{ - FilePersistence: &feastdevv1.OnlineStoreFilePersistence{ - Path: "/data/online.db", + Expect(err.Error()).To(ContainSubstring("online store")) + }) + + It("should reject replicas and autoscaling set simultaneously", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: "cel-mutual-excl", Namespace: "default"}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "celtest", + Replicas: ptr(int32(3)), + Services: &feastdevv1.FeatureStoreServices{ + Scaling: &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{MaxReplicas: 5}, + }, + OnlineStore: dbOnlineStore, + Registry: dbRegistry, + }, }, } - Expect(feast.validateScaling()).To(Succeed()) + err := k8sClient.Create(ctx, fs) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("mutually exclusive")) }) }) Describe("getDesiredReplicas", func() { - It("should return nil when no scaling is configured", func() { - Expect(feast.getDesiredReplicas()).To(BeNil()) + It("should return 1 when no explicit replicas are configured (default)", func() { + replicas := feast.getDesiredReplicas() + Expect(replicas).NotTo(BeNil()) + Expect(*replicas).To(Equal(int32(1))) }) It("should return static replicas when configured", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } + featureStore.Status.Applied.Replicas = ptr(int32(3)) replicas := feast.getDesiredReplicas() Expect(replicas).NotTo(BeNil()) Expect(*replicas).To(Equal(int32(3))) @@ -486,18 +458,14 @@ var _ = Describe("Horizontal Scaling", func() { Expect(strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) - It("should default to RollingUpdate when scaling is enabled", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } + It("should default to RollingUpdate when scaling is enabled via replicas", func() { + featureStore.Status.Applied.Replicas = ptr(int32(3)) strategy := feast.getDeploymentStrategy() Expect(strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) }) It("should respect user-defined strategy even with scaling", func() { - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } + featureStore.Status.Applied.Replicas = ptr(int32(3)) featureStore.Status.Applied.Services.DeploymentStrategy = &appsv1.DeploymentStrategy{ Type: appsv1.RecreateDeploymentStrategyType, } @@ -507,8 +475,6 @@ var _ = Describe("Horizontal Scaling", func() { }) Describe("setDeployment with scaling", func() { - // These tests use file-based (default) persistence to avoid needing real secrets - // for the feature_store.yaml generation that happens inside setDeployment. setFilePersistence := func() { featureStore.Status.Applied.Services.OnlineStore = &feastdevv1.OnlineStore{ Server: &feastdevv1.ServerConfigs{ @@ -547,11 +513,7 @@ var _ = Describe("Horizontal Scaling", func() { It("should set static replicas on the deployment", func() { setFilePersistence() - featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ - Replicas: ptr(int32(3)), - } - Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) - feast.refreshFeatureStore(ctx, typeNamespacedName) + featureStore.Status.Applied.Replicas = ptr(int32(3)) deployment := feast.initFeastDeploy() Expect(feast.setDeployment(deployment)).To(Succeed()) @@ -566,8 +528,6 @@ var _ = Describe("Horizontal Scaling", func() { MaxReplicas: 5, }, } - Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) - feast.refreshFeatureStore(ctx, typeNamespacedName) deployment := feast.initFeastDeploy() existing := int32(4) @@ -577,17 +537,15 @@ var _ = Describe("Horizontal Scaling", func() { Expect(*deployment.Spec.Replicas).To(Equal(int32(4))) }) - It("should preserve existing replicas when no scaling is configured", func() { + It("should set default replicas=1 when no explicit scaling is configured", func() { setFilePersistence() Expect(k8sClient.Status().Update(ctx, featureStore)).To(Succeed()) feast.refreshFeatureStore(ctx, typeNamespacedName) deployment := feast.initFeastDeploy() - existing := int32(2) - deployment.Spec.Replicas = &existing Expect(feast.setDeployment(deployment)).To(Succeed()) Expect(deployment.Spec.Replicas).NotTo(BeNil()) - Expect(*deployment.Spec.Replicas).To(Equal(int32(2))) + Expect(*deployment.Spec.Replicas).To(Equal(int32(1))) }) }) @@ -635,4 +593,94 @@ var _ = Describe("Horizontal Scaling", func() { Expect(hpa.Spec.ScaleTargetRef.Name).To(Equal(GetFeastName(featureStore))) }) }) + + Describe("Scale sub-resource", func() { + newDBFeatureStore := func(name string) *feastdevv1.FeatureStore { + return &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "scaletest", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{ + Persistence: &feastdevv1.OnlineStorePersistence{ + DBPersistence: &feastdevv1.OnlineStoreDBStorePersistence{ + Type: "redis", + SecretRef: corev1.LocalObjectReference{Name: "redis-secret"}, + }, + }, + }, + Registry: &feastdevv1.Registry{ + Local: &feastdevv1.LocalRegistryConfig{ + Persistence: &feastdevv1.RegistryPersistence{ + DBPersistence: &feastdevv1.RegistryDBStorePersistence{ + Type: "sql", + SecretRef: corev1.LocalObjectReference{Name: "registry-secret"}, + }, + }, + }, + }, + }, + }, + } + } + + It("should allow scaling up via the scale sub-resource with DB persistence", func() { + fs := newDBFeatureStore("scale-sub-valid") + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + defer func() { Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }() + + scale := &autoscalingv1.Scale{} + Expect(k8sClient.SubResource("scale").Get(ctx, fs, scale)).To(Succeed()) + Expect(scale.Spec.Replicas).To(Equal(int32(1))) + + scale.Spec.Replicas = 3 + Expect(k8sClient.SubResource("scale").Update(ctx, fs, client.WithSubResourceBody(scale))).To(Succeed()) + + updated := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fs.Name, Namespace: fs.Namespace}, updated)).To(Succeed()) + Expect(updated.Spec.Replicas).NotTo(BeNil()) + Expect(*updated.Spec.Replicas).To(Equal(int32(3))) + }) + + It("should reject scaling up via the scale sub-resource without DB persistence", func() { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scale-sub-reject", + Namespace: "default", + }, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "scaletest", + }, + } + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + defer func() { Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }() + + scale := &autoscalingv1.Scale{} + Expect(k8sClient.SubResource("scale").Get(ctx, fs, scale)).To(Succeed()) + + scale.Spec.Replicas = 3 + err := k8sClient.SubResource("scale").Update(ctx, fs, client.WithSubResourceBody(scale)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("online store")) + }) + + It("should read the status replicas from the scale sub-resource", func() { + fs := newDBFeatureStore("scale-sub-status") + fs.Spec.Replicas = ptr(int32(2)) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + defer func() { Expect(k8sClient.Delete(ctx, fs)).To(Succeed()) }() + + fs.Status.Replicas = 2 + fs.Status.Applied.FeastProject = fs.Spec.FeastProject + Expect(k8sClient.Status().Update(ctx, fs)).To(Succeed()) + + scale := &autoscalingv1.Scale{} + Expect(k8sClient.SubResource("scale").Get(ctx, fs, scale)).To(Succeed()) + Expect(scale.Status.Replicas).To(Equal(int32(2))) + Expect(scale.Spec.Replicas).To(Equal(int32(2))) + }) + }) }) diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 3d5e2ddf37f..a76f21d18c8 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -75,9 +75,6 @@ func (feast *FeastServices) Deploy() error { return err } - if err := feast.validateScaling(); err != nil { - return err - } if err := feast.createServiceAccount(); err != nil { return err } @@ -97,8 +94,6 @@ func (feast *FeastServices) Deploy() error { return err } - feast.updateScalingStatus() - return nil } @@ -352,6 +347,8 @@ func (feast *FeastServices) createDeployment() error { logger.Info("Successfully reconciled", "Deployment", deploy.Name, "operation", op) } + feast.updateScalingStatus(deploy) + return nil } @@ -397,9 +394,8 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { cr := feast.Handler.FeatureStore // Determine replica count: - // - Static replicas from scaling config take precedence - // - When HPA is configured, preserve existing value so the HPA controller manages it - // - Otherwise preserve the existing value (supports external autoscalers) + // - spec.replicas is set on the Deployment (defaults to 1) + // - When HPA is configured, replicas is left unset so the HPA controller manages it replicas := deploy.Spec.Replicas if desired := feast.getDesiredReplicas(); desired != nil { replicas = desired diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index d3ee774a427..9ce1ecd749a 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -496,61 +496,11 @@ func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1.F // isScalingEnabled returns true when the user has configured horizontal scaling // with either static replicas > 1 or HPA autoscaling. func isScalingEnabled(featureStore *feastdevv1.FeatureStore) bool { - services := featureStore.Status.Applied.Services - if services == nil || services.Scaling == nil { - return false - } - scaling := services.Scaling - if scaling.Replicas != nil && *scaling.Replicas > 1 { + if featureStore.Status.Applied.Replicas != nil && *featureStore.Status.Applied.Replicas > 1 { return true } - return scaling.Autoscaling != nil -} - -// isFilePersistence returns true if any enabled service uses file-based persistence -// (SQLite, DuckDB, Dask, registry.db, etc.) rather than an external database. -func isFilePersistence(featureStore *feastdevv1.FeatureStore) bool { services := featureStore.Status.Applied.Services - if services == nil { - return false - } - - if services.OnlineStore != nil && services.OnlineStore.Persistence != nil && - services.OnlineStore.Persistence.FilePersistence != nil { - return true - } - - if services.OfflineStore != nil && services.OfflineStore.Persistence != nil && - services.OfflineStore.Persistence.FilePersistence != nil { - return true - } - - // When no registry is configured, the deployment defaults to a file-based - // registry (registry.db). Only a remote registry or an explicit local - // registry with DB persistence is safe for multi-replica. - if services.Registry == nil { - return true - } - if isRemoteRegistry(featureStore) { - return false - } - if IsLocalRegistry(featureStore) { - if services.Registry.Local.Persistence == nil || - services.Registry.Local.Persistence.DBPersistence == nil { - // S3/GS-backed registry file persistence is safe for multi-replica - if services.Registry.Local.Persistence != nil && - services.Registry.Local.Persistence.FilePersistence != nil && - services.Registry.Local.Persistence.FilePersistence.Path != "" { - path := services.Registry.Local.Persistence.FilePersistence.Path - if strings.HasPrefix(path, "s3://") || strings.HasPrefix(path, "gs://") { - return false - } - } - return true - } - } - - return false + return services != nil && services.Scaling != nil && services.Scaling.Autoscaling != nil } func boolPtr(value bool) *bool { diff --git a/infra/website/docs/blog/scaling-feast-feature-server.md b/infra/website/docs/blog/scaling-feast-feature-server.md index 78fcf5154dd..4406d280c81 100644 --- a/infra/website/docs/blog/scaling-feast-feature-server.md +++ b/infra/website/docs/blog/scaling-feast-feature-server.md @@ -24,11 +24,11 @@ Teams have been manually patching Deployments or creating external HPAs, but thi # The Solution: Native Scaling Support -The Feast Operator now supports three scaling modes through a new `scaling` field in the FeatureStore CR: +The Feast Operator now supports three scaling modes. The FeatureStore CRD implements the Kubernetes **scale sub-resource**, which means you can also scale with `kubectl scale featurestore/my-feast --replicas=3`. ## 1. Static Replicas -The simplest approach — set a fixed number of replicas: +The simplest approach — set a fixed number of replicas via `spec.replicas`: ```yaml apiVersion: feast.dev/v1 @@ -37,9 +37,8 @@ metadata: name: production-feast spec: feastProject: my_project + replicas: 3 services: - scaling: - replicas: 3 onlineStore: persistence: store: @@ -59,7 +58,7 @@ This gives you high availability and load distribution with a predictable resour ## 2. HPA Autoscaling -For workloads with variable traffic patterns, the operator can create and manage a `HorizontalPodAutoscaler` directly: +For workloads with variable traffic patterns, the operator can create and manage a `HorizontalPodAutoscaler` directly. HPA autoscaling is configured under `services.scaling.autoscaling` and is mutually exclusive with `spec.replicas > 1`: ```yaml apiVersion: feast.dev/v1 @@ -107,9 +106,9 @@ The operator creates the HPA as an owned resource — it's automatically cleaned ## 3. External Autoscalers (KEDA, Custom HPAs) -For teams using [KEDA](https://keda.sh) or other external autoscalers, the operator is designed to stay out of the way. When no `scaling` field is set, the operator preserves whatever replica count an external controller sets on the Deployment. +For teams using [KEDA](https://keda.sh) or other external autoscalers, KEDA should target the FeatureStore's scale sub-resource directly (since it implements the Kubernetes scale API). This is the recommended approach because the operator manages the Deployment's replica count from `spec.replicas` — targeting the Deployment directly would conflict with the operator's reconciliation. -To use KEDA, configure the FeatureStore with an explicit `RollingUpdate` strategy and DB-backed persistence, then create a KEDA `ScaledObject` targeting the Feast Deployment: +When using KEDA, do **not** set `spec.replicas > 1` or `services.scaling.autoscaling` — KEDA manages the replica count through the scale sub-resource. Configure the FeatureStore with DB-backed persistence, then create a KEDA `ScaledObject` targeting the FeatureStore resource: ```yaml apiVersion: feast.dev/v1 @@ -119,8 +118,6 @@ metadata: spec: feastProject: my_project services: - deploymentStrategy: - type: RollingUpdate onlineStore: persistence: store: @@ -141,8 +138,10 @@ metadata: name: feast-scaledobject spec: scaleTargetRef: - name: feast-keda-feast # matches the Feast deployment name - minReplicaCount: 2 + apiVersion: feast.dev/v1 + kind: FeatureStore + name: keda-feast + minReplicaCount: 1 maxReplicaCount: 10 triggers: - type: prometheus @@ -153,21 +152,20 @@ spec: threshold: "100" ``` -This gives you the full power of KEDA's 50+ event-driven triggers while the operator manages the rest of the Feast deployment lifecycle. +When KEDA scales up `spec.replicas` via the scale sub-resource, the CRD's CEL validation rules automatically ensure DB-backed persistence is configured. The operator also automatically switches the deployment strategy to `RollingUpdate` when `replicas > 1`. This gives you the full power of KEDA's 50+ event-driven triggers with built-in safety checks. # Safety First: Persistence Validation Not all persistence backends are safe for multi-replica deployments. File-based stores like SQLite, DuckDB, and local `registry.db` use single-writer file locks that don't work across pods. -The operator enforces this at reconciliation time — if you configure scaling with file-based persistence, you'll get a clear error: +The operator enforces this at admission time via CEL validation rules on the CRD — if you try to create or update a FeatureStore with scaling and file-based persistence, the API server rejects the request immediately: ``` -horizontal scaling (replicas > 1 or autoscaling) requires DB-backed persistence -for all enabled services. File-based persistence (SQLite, DuckDB, registry.db) -is incompatible with multiple replicas +Scaling requires DB-backed persistence for the online store. +Configure services.onlineStore.persistence.store when using replicas > 1 or autoscaling. ``` -This validation applies to all enabled services (online store, offline store, and registry). Object-store-backed registry paths (`s3://` and `gs://`) are treated as safe since they support concurrent readers. +This validation applies to all enabled services (online store, offline store, and registry) and is enforced for both direct CR updates and `kubectl scale` commands via the scale sub-resource. Object-store-backed registry paths (`s3://` and `gs://`) are treated as safe since they support concurrent readers. | Persistence Type | Compatible with Scaling? | |---|---| @@ -184,7 +182,7 @@ This validation applies to all enabled services (online store, offline store, an The implementation adds three key behaviors to the operator's reconciliation loop: -**1. Replica management** — When static replicas are configured, the operator sets them on the Deployment. When HPA is configured, the operator leaves the `replicas` field unset so the HPA controller can manage it. When neither is configured, existing replicas are preserved (supporting external autoscalers). +**1. Replica management** — The operator sets the Deployment's replica count from `spec.replicas` (which defaults to 1). When HPA is configured, the operator leaves the `replicas` field unset so the HPA controller can manage it. External autoscalers like KEDA can update the replica count through the FeatureStore's scale sub-resource, which updates `spec.replicas` and triggers the operator to reconcile. **2. Deployment strategy** — The operator automatically switches from `Recreate` (the default for single-replica) to `RollingUpdate` when scaling is enabled. This prevents the "kill-all-pods-then-start-new-ones" behavior that would cause downtime during scaling events. Users can always override this with an explicit `deploymentStrategy` in the CR. @@ -211,16 +209,17 @@ Scaling is designed to work seamlessly with existing operator features: **1. Ensure DB-backed persistence** for all enabled services (online store, offline store, registry). -**2. Add a `scaling` field** to your FeatureStore CR: +**2. Configure scaling** in your FeatureStore CR — use either static replicas or HPA (mutually exclusive): ```yaml -services: - scaling: - replicas: 3 # static replicas - # -- OR -- - # autoscaling: # HPA - # minReplicas: 2 - # maxReplicas: 10 +spec: + replicas: 3 # static replicas (top-level) + # -- OR -- + # services: + # scaling: + # autoscaling: # HPA + # minReplicas: 2 + # maxReplicas: 10 ``` **3. Apply** the updated CR: From 101dd55571a51d1c4a559abba3c152453b4402ed Mon Sep 17 00:00:00 2001 From: ntkathole Date: Thu, 26 Feb 2026 19:55:57 +0530 Subject: [PATCH 4/5] fix: Server side apply Signed-off-by: ntkathole --- .../internal/controller/services/scaling.go | 90 ++++++++++--------- .../controller/services/scaling_test.go | 28 ++++-- 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/infra/feast-operator/internal/controller/services/scaling.go b/infra/feast-operator/internal/controller/services/scaling.go index 7649a31d22b..6041ce53862 100644 --- a/infra/feast-operator/internal/controller/services/scaling.go +++ b/infra/feast-operator/internal/controller/services/scaling.go @@ -22,13 +22,14 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) const ( defaultHPACPUUtilization int32 = 80 defaultHPAMinReplicas int32 = 1 + fieldManager = "feast-operator" ) // getDesiredReplicas returns the replica count the operator should set on the @@ -49,45 +50,35 @@ func (feast *FeastServices) getDesiredReplicas() *int32 { } // createOrDeleteHPA reconciles the HorizontalPodAutoscaler for the FeatureStore -// deployment. If autoscaling is not configured, any existing HPA is deleted. +// deployment using Server-Side Apply. If autoscaling is not configured, any +// existing HPA is deleted. func (feast *FeastServices) createOrDeleteHPA() error { cr := feast.Handler.FeatureStore - hpa := feast.initHPA() scaling := cr.Status.Applied.Services.Scaling if scaling == nil || scaling.Autoscaling == nil { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: feast.GetObjectMeta(), + } + hpa.SetGroupVersionKind(autoscalingv2.SchemeGroupVersion.WithKind("HorizontalPodAutoscaler")) return feast.Handler.DeleteOwnedFeastObj(hpa) } + hpa := feast.buildHPA() logger := log.FromContext(feast.Handler.Context) - if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, hpa, controllerutil.MutateFn(func() error { - return feast.setHPA(hpa) - })); err != nil { + if err := feast.Handler.Client.Patch(feast.Handler.Context, hpa, + client.Apply, client.FieldOwner(fieldManager), client.ForceOwnership); err != nil { return err - } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { - logger.Info("Successfully reconciled", "HorizontalPodAutoscaler", hpa.Name, "operation", op) } + logger.Info("Successfully applied", "HorizontalPodAutoscaler", hpa.Name) return nil } -func (feast *FeastServices) initHPA() *autoscalingv2.HorizontalPodAutoscaler { - hpa := &autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: feast.GetObjectMeta(), - } - hpa.SetGroupVersionKind(autoscalingv2.SchemeGroupVersion.WithKind("HorizontalPodAutoscaler")) - return hpa -} - -func (feast *FeastServices) setHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) error { +// buildHPA constructs the fully desired HPA state for Server-Side Apply. +func (feast *FeastServices) buildHPA() *autoscalingv2.HorizontalPodAutoscaler { cr := feast.Handler.FeatureStore - scaling := cr.Status.Applied.Services.Scaling - if scaling == nil || scaling.Autoscaling == nil { - return nil - } - autoscaling := scaling.Autoscaling - - hpa.Labels = feast.getLabels() + autoscaling := cr.Status.Applied.Services.Scaling.Autoscaling deploy := feast.initFeastDeploy() minReplicas := defaultHPAMinReplicas @@ -95,27 +86,46 @@ func (feast *FeastServices) setHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) e minReplicas = *autoscaling.MinReplicas } - hpa.Spec = autoscalingv2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: deploy.Name, - }, - MinReplicas: &minReplicas, - MaxReplicas: autoscaling.MaxReplicas, - } - + metrics := defaultHPAMetrics() if len(autoscaling.Metrics) > 0 { - hpa.Spec.Metrics = autoscaling.Metrics - } else { - hpa.Spec.Metrics = defaultHPAMetrics() + metrics = autoscaling.Metrics } - if autoscaling.Behavior != nil { - hpa.Spec.Behavior = autoscaling.Behavior + isController := true + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: autoscalingv2.SchemeGroupVersion.String(), + Kind: "HorizontalPodAutoscaler", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetObjectMeta().Name, + Namespace: feast.GetObjectMeta().Namespace, + Labels: feast.getLabels(), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: feastdevv1.GroupVersion.String(), + Kind: "FeatureStore", + Name: cr.Name, + UID: cr.UID, + Controller: &isController, + BlockOwnerDeletion: &isController, + }, + }, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: deploy.Name, + }, + MinReplicas: &minReplicas, + MaxReplicas: autoscaling.MaxReplicas, + Metrics: metrics, + Behavior: autoscaling.Behavior, + }, } - return controllerutil.SetControllerReference(cr, hpa, feast.Handler.Scheme) + return hpa } func defaultHPAMetrics() []autoscalingv2.MetricSpec { diff --git a/infra/feast-operator/internal/controller/services/scaling_test.go b/infra/feast-operator/internal/controller/services/scaling_test.go index dedb780f7e5..eb6ce6ba251 100644 --- a/infra/feast-operator/internal/controller/services/scaling_test.go +++ b/infra/feast-operator/internal/controller/services/scaling_test.go @@ -550,22 +550,21 @@ var _ = Describe("Horizontal Scaling", func() { }) Describe("HPA Configuration", func() { - It("should create an HPA with default CPU metrics", func() { + It("should build an HPA with default CPU metrics", func() { featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ Autoscaling: &feastdevv1.AutoscalingConfig{ MaxReplicas: 10, }, } - hpa := feast.initHPA() - Expect(feast.setHPA(hpa)).To(Succeed()) + hpa := feast.buildHPA() Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1))) Expect(hpa.Spec.Metrics).To(HaveLen(1)) Expect(hpa.Spec.Metrics[0].Resource.Name).To(Equal(corev1.ResourceCPU)) }) - It("should create an HPA with custom min replicas", func() { + It("should build an HPA with custom min replicas", func() { featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ Autoscaling: &feastdevv1.AutoscalingConfig{ MinReplicas: ptr(int32(2)), @@ -573,8 +572,7 @@ var _ = Describe("Horizontal Scaling", func() { }, } - hpa := feast.initHPA() - Expect(feast.setHPA(hpa)).To(Succeed()) + hpa := feast.buildHPA() Expect(*hpa.Spec.MinReplicas).To(Equal(int32(2))) Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) }) @@ -586,12 +584,26 @@ var _ = Describe("Horizontal Scaling", func() { }, } - hpa := feast.initHPA() - Expect(feast.setHPA(hpa)).To(Succeed()) + hpa := feast.buildHPA() Expect(hpa.Spec.ScaleTargetRef.APIVersion).To(Equal("apps/v1")) Expect(hpa.Spec.ScaleTargetRef.Kind).To(Equal("Deployment")) Expect(hpa.Spec.ScaleTargetRef.Name).To(Equal(GetFeastName(featureStore))) }) + + It("should set TypeMeta and owner reference for SSA", func() { + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 5, + }, + } + + hpa := feast.buildHPA() + Expect(hpa.TypeMeta.APIVersion).To(Equal("autoscaling/v2")) + Expect(hpa.TypeMeta.Kind).To(Equal("HorizontalPodAutoscaler")) + Expect(hpa.OwnerReferences).To(HaveLen(1)) + Expect(hpa.OwnerReferences[0].Name).To(Equal(featureStore.Name)) + Expect(*hpa.OwnerReferences[0].Controller).To(BeTrue()) + }) }) Describe("Scale sub-resource", func() { From befdf231d609b0fb4a14ad766092a4e05d8a8cd8 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Thu, 26 Feb 2026 21:01:56 +0530 Subject: [PATCH 5/5] fix: Use autoscalingv2.HorizontalPodAutoscaler Signed-off-by: ntkathole --- infra/feast-operator/config/rbac/role.yaml | 1 + infra/feast-operator/dist/install.yaml | 1 + .../controller/featurestore_controller.go | 2 +- .../internal/controller/services/scaling.go | 147 +++++++++++------- .../controller/services/scaling_test.go | 57 +++++-- 5 files changed, 136 insertions(+), 72 deletions(-) diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index ded745f1631..3fa228afc6b 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -30,6 +30,7 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 38445ebfc1a..0c0b05be388 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -18585,6 +18585,7 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index b0f709cc157..d73b30c0175 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -66,7 +66,7 @@ type FeatureStoreReconciler struct { // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/infra/feast-operator/internal/controller/services/scaling.go b/infra/feast-operator/internal/controller/services/scaling.go index 6041ce53862..b8555555498 100644 --- a/infra/feast-operator/internal/controller/services/scaling.go +++ b/infra/feast-operator/internal/controller/services/scaling.go @@ -17,11 +17,16 @@ limitations under the License. package services import ( + "encoding/json" + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + hpaac "k8s.io/client-go/applyconfigurations/autoscaling/v2" + metaac "k8s.io/client-go/applyconfigurations/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -50,8 +55,8 @@ func (feast *FeastServices) getDesiredReplicas() *int32 { } // createOrDeleteHPA reconciles the HorizontalPodAutoscaler for the FeatureStore -// deployment using Server-Side Apply. If autoscaling is not configured, any -// existing HPA is deleted. +// deployment using Server-Side Apply with typed apply configurations. If +// autoscaling is not configured, any existing HPA is deleted. func (feast *FeastServices) createOrDeleteHPA() error { cr := feast.Handler.FeatureStore @@ -64,10 +69,17 @@ func (feast *FeastServices) createOrDeleteHPA() error { return feast.Handler.DeleteOwnedFeastObj(hpa) } - hpa := feast.buildHPA() + hpaAC := feast.buildHPAApplyConfig() + data, err := json.Marshal(hpaAC) + if err != nil { + return err + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: feast.GetObjectMeta()} logger := log.FromContext(feast.Handler.Context) if err := feast.Handler.Client.Patch(feast.Handler.Context, hpa, - client.Apply, client.FieldOwner(fieldManager), client.ForceOwnership); err != nil { + client.RawPatch(types.ApplyPatchType, data), + client.FieldOwner(fieldManager), client.ForceOwnership); err != nil { return err } logger.Info("Successfully applied", "HorizontalPodAutoscaler", hpa.Name) @@ -75,73 +87,96 @@ func (feast *FeastServices) createOrDeleteHPA() error { return nil } -// buildHPA constructs the fully desired HPA state for Server-Side Apply. -func (feast *FeastServices) buildHPA() *autoscalingv2.HorizontalPodAutoscaler { +// buildHPAApplyConfig constructs the fully desired HPA state as a typed apply +// configuration for Server-Side Apply. +func (feast *FeastServices) buildHPAApplyConfig() *hpaac.HorizontalPodAutoscalerApplyConfiguration { cr := feast.Handler.FeatureStore autoscaling := cr.Status.Applied.Services.Scaling.Autoscaling - + objMeta := feast.GetObjectMeta() deploy := feast.initFeastDeploy() + minReplicas := defaultHPAMinReplicas if autoscaling.MinReplicas != nil { minReplicas = *autoscaling.MinReplicas } - metrics := defaultHPAMetrics() + hpa := hpaac.HorizontalPodAutoscaler(objMeta.Name, objMeta.Namespace). + WithLabels(feast.getLabels()). + WithOwnerReferences( + metaac.OwnerReference(). + WithAPIVersion(feastdevv1.GroupVersion.String()). + WithKind("FeatureStore"). + WithName(cr.Name). + WithUID(cr.UID). + WithController(true). + WithBlockOwnerDeletion(true), + ). + WithSpec(hpaac.HorizontalPodAutoscalerSpec(). + WithScaleTargetRef( + hpaac.CrossVersionObjectReference(). + WithAPIVersion(appsv1.SchemeGroupVersion.String()). + WithKind("Deployment"). + WithName(deploy.Name), + ). + WithMinReplicas(minReplicas). + WithMaxReplicas(autoscaling.MaxReplicas), + ) + if len(autoscaling.Metrics) > 0 { - metrics = autoscaling.Metrics - } - - isController := true - hpa := &autoscalingv2.HorizontalPodAutoscaler{ - TypeMeta: metav1.TypeMeta{ - APIVersion: autoscalingv2.SchemeGroupVersion.String(), - Kind: "HorizontalPodAutoscaler", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: feast.GetObjectMeta().Name, - Namespace: feast.GetObjectMeta().Namespace, - Labels: feast.getLabels(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: feastdevv1.GroupVersion.String(), - Kind: "FeatureStore", - Name: cr.Name, - UID: cr.UID, - Controller: &isController, - BlockOwnerDeletion: &isController, - }, - }, - }, - Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - APIVersion: appsv1.SchemeGroupVersion.String(), - Kind: "Deployment", - Name: deploy.Name, - }, - MinReplicas: &minReplicas, - MaxReplicas: autoscaling.MaxReplicas, - Metrics: metrics, - Behavior: autoscaling.Behavior, - }, + hpa.Spec.Metrics = convertMetrics(autoscaling.Metrics) + } else { + hpa.Spec.Metrics = defaultHPAMetrics() + } + + if autoscaling.Behavior != nil { + hpa.Spec.Behavior = convertBehavior(autoscaling.Behavior) } return hpa } -func defaultHPAMetrics() []autoscalingv2.MetricSpec { - utilization := defaultHPACPUUtilization - return []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricSource{ - Name: corev1.ResourceCPU, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: &utilization, - }, - }, - }, +func defaultHPAMetrics() []hpaac.MetricSpecApplyConfiguration { + return []hpaac.MetricSpecApplyConfiguration{ + *hpaac.MetricSpec(). + WithType(autoscalingv2.ResourceMetricSourceType). + WithResource( + hpaac.ResourceMetricSource(). + WithName(corev1.ResourceCPU). + WithTarget( + hpaac.MetricTarget(). + WithType(autoscalingv2.UtilizationMetricType). + WithAverageUtilization(defaultHPACPUUtilization), + ), + ), + } +} + +// convertMetrics converts standard API metric specs to their apply configuration +// equivalents via JSON round-trip (the types share identical JSON schemas). +func convertMetrics(metrics []autoscalingv2.MetricSpec) []hpaac.MetricSpecApplyConfiguration { + data, err := json.Marshal(metrics) + if err != nil { + return nil + } + var result []hpaac.MetricSpecApplyConfiguration + if err := json.Unmarshal(data, &result); err != nil { + return nil + } + return result +} + +// convertBehavior converts a standard API behavior spec to its apply configuration +// equivalent via JSON round-trip. +func convertBehavior(behavior *autoscalingv2.HorizontalPodAutoscalerBehavior) *hpaac.HorizontalPodAutoscalerBehaviorApplyConfiguration { + data, err := json.Marshal(behavior) + if err != nil { + return nil + } + result := &hpaac.HorizontalPodAutoscalerBehaviorApplyConfiguration{} + if err := json.Unmarshal(data, result); err != nil { + return nil } + return result } // updateScalingStatus updates the scaling status fields using the deployment diff --git a/infra/feast-operator/internal/controller/services/scaling_test.go b/infra/feast-operator/internal/controller/services/scaling_test.go index eb6ce6ba251..db803757112 100644 --- a/infra/feast-operator/internal/controller/services/scaling_test.go +++ b/infra/feast-operator/internal/controller/services/scaling_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -550,21 +551,21 @@ var _ = Describe("Horizontal Scaling", func() { }) Describe("HPA Configuration", func() { - It("should build an HPA with default CPU metrics", func() { + It("should build an HPA apply config with default CPU metrics", func() { featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ Autoscaling: &feastdevv1.AutoscalingConfig{ MaxReplicas: 10, }, } - hpa := feast.buildHPA() - Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) + hpa := feast.buildHPAApplyConfig() + Expect(*hpa.Spec.MaxReplicas).To(Equal(int32(10))) Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1))) Expect(hpa.Spec.Metrics).To(HaveLen(1)) - Expect(hpa.Spec.Metrics[0].Resource.Name).To(Equal(corev1.ResourceCPU)) + Expect(*hpa.Spec.Metrics[0].Resource.Name).To(Equal(corev1.ResourceCPU)) }) - It("should build an HPA with custom min replicas", func() { + It("should build an HPA apply config with custom min replicas", func() { featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ Autoscaling: &feastdevv1.AutoscalingConfig{ MinReplicas: ptr(int32(2)), @@ -572,9 +573,9 @@ var _ = Describe("Horizontal Scaling", func() { }, } - hpa := feast.buildHPA() + hpa := feast.buildHPAApplyConfig() Expect(*hpa.Spec.MinReplicas).To(Equal(int32(2))) - Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10))) + Expect(*hpa.Spec.MaxReplicas).To(Equal(int32(10))) }) It("should set correct scale target reference", func() { @@ -584,10 +585,10 @@ var _ = Describe("Horizontal Scaling", func() { }, } - hpa := feast.buildHPA() - Expect(hpa.Spec.ScaleTargetRef.APIVersion).To(Equal("apps/v1")) - Expect(hpa.Spec.ScaleTargetRef.Kind).To(Equal("Deployment")) - Expect(hpa.Spec.ScaleTargetRef.Name).To(Equal(GetFeastName(featureStore))) + hpa := feast.buildHPAApplyConfig() + Expect(*hpa.Spec.ScaleTargetRef.APIVersion).To(Equal("apps/v1")) + Expect(*hpa.Spec.ScaleTargetRef.Kind).To(Equal("Deployment")) + Expect(*hpa.Spec.ScaleTargetRef.Name).To(Equal(GetFeastName(featureStore))) }) It("should set TypeMeta and owner reference for SSA", func() { @@ -597,13 +598,39 @@ var _ = Describe("Horizontal Scaling", func() { }, } - hpa := feast.buildHPA() - Expect(hpa.TypeMeta.APIVersion).To(Equal("autoscaling/v2")) - Expect(hpa.TypeMeta.Kind).To(Equal("HorizontalPodAutoscaler")) + hpa := feast.buildHPAApplyConfig() + Expect(*hpa.Kind).To(Equal("HorizontalPodAutoscaler")) + Expect(*hpa.APIVersion).To(Equal("autoscaling/v2")) Expect(hpa.OwnerReferences).To(HaveLen(1)) - Expect(hpa.OwnerReferences[0].Name).To(Equal(featureStore.Name)) + Expect(*hpa.OwnerReferences[0].Name).To(Equal(featureStore.Name)) Expect(*hpa.OwnerReferences[0].Controller).To(BeTrue()) }) + + It("should convert custom metrics via JSON round-trip", func() { + customMetrics := []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr(int32(75)), + }, + }, + }, + } + featureStore.Status.Applied.Services.Scaling = &feastdevv1.ScalingConfig{ + Autoscaling: &feastdevv1.AutoscalingConfig{ + MaxReplicas: 10, + Metrics: customMetrics, + }, + } + + hpa := feast.buildHPAApplyConfig() + Expect(hpa.Spec.Metrics).To(HaveLen(1)) + Expect(*hpa.Spec.Metrics[0].Resource.Name).To(Equal(corev1.ResourceMemory)) + Expect(*hpa.Spec.Metrics[0].Resource.Target.AverageUtilization).To(Equal(int32(75))) + }) }) Describe("Scale sub-resource", func() {