Skip to content

Commit bf1e54a

Browse files
ntkatholentkathole
committed
feat: Support Lineage configs via operator
Signed-off-by: ntkathole <nikhilkathole2683@gmail.com>
1 parent e79bd33 commit bf1e54a

34 files changed

Lines changed: 4831 additions & 173 deletions

‎.secrets.baseline‎

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/reference/openlineage.md‎

Lines changed: 389 additions & 29 deletions
Large diffs are not rendered by default.

‎infra/feast-operator/api/v1/featurestore_types.go‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const (
3737
OnlineStoreReadyType = "OnlineStore"
3838
RegistryReadyType = "Registry"
3939
UIReadyType = "UI"
40+
LineageReadyType = "Lineage"
4041
ReadyType = "FeatureStore"
4142
AuthorizationReadyType = "Authorization"
4243
CronJobReadyType = "CronJob"
@@ -49,6 +50,7 @@ const (
4950
OnlineStoreFailedReason = "OnlineStoreDeploymentFailed"
5051
RegistryFailedReason = "RegistryDeploymentFailed"
5152
UIFailedReason = "UIDeploymentFailed"
53+
LineageFailedReason = "LineageDeploymentFailed"
5254
ClientFailedReason = "ClientDeploymentFailed"
5355
CronJobFailedReason = "CronJobDeploymentFailed"
5456
KubernetesAuthzFailedReason = "KubernetesAuthorizationDeploymentFailed"
@@ -60,6 +62,7 @@ const (
6062
OnlineStoreReadyMessage = "Online Store installation complete"
6163
RegistryReadyMessage = "Registry installation complete"
6264
UIReadyMessage = "UI installation complete"
65+
LineageReadyMessage = "Lineage server installation complete"
6366
ClientReadyMessage = "Client installation complete"
6467
CronJobReadyMessage = "CronJob installation complete"
6568
KubernetesAuthzReadyMessage = "Kubernetes authorization installation complete"
@@ -93,7 +96,7 @@ type OpenLineageConfig struct {
9396
// +kubebuilder:validation:Enum=http;console;file;kafka
9497
// +optional
9598
TransportType *string `json:"transportType,omitempty"`
96-
// URL for HTTP transport (e.g. http://marquez:5000). Required when transportType is "http".
99+
// URL for HTTP transport (e.g. http://feast-example-lineage:6580). Required when transportType is "http".
97100
// +optional
98101
TransportUrl *string `json:"transportUrl,omitempty"`
99102
// API endpoint path appended to transportUrl. Defaults to "api/v1/lineage".
@@ -140,6 +143,35 @@ type OpenLineageConsumerConfig struct {
140143
// RBAC-based filtering of lineage data in the UI.
141144
// +optional
142145
NamespaceMapping map[string]string `json:"namespaceMapping,omitempty"`
146+
// RetentionDays is the number of days to retain OpenLineage events and runs.
147+
// Events older than this are automatically pruned. Set to 0 to disable pruning.
148+
// +kubebuilder:default=30
149+
// +kubebuilder:validation:Minimum=0
150+
// +optional
151+
RetentionDays *int32 `json:"retentionDays,omitempty"`
152+
// RetentionCheckIntervalHours is how often the background pruning task runs, in hours.
153+
// +kubebuilder:default=6
154+
// +kubebuilder:validation:Minimum=1
155+
// +optional
156+
RetentionCheckIntervalHours *int32 `json:"retentionCheckIntervalHours,omitempty"`
157+
// LineageServer enables a separate Deployment for the OpenLineage consumer.
158+
// When set, the consumer is removed from the UI/registry Pod and runs
159+
// independently with its own scaling. The Feast producer transport is
160+
// auto-configured to send events to the lineage Service.
161+
// +optional
162+
LineageServer *LineageServerConfig `json:"lineageServer,omitempty"`
163+
}
164+
165+
// LineageServerConfig defines the separate lineage server Deployment.
166+
type LineageServerConfig struct {
167+
// Server configuration (image, resources, env, TLS, etc.).
168+
// +optional
169+
Server *ServerConfigs `json:"server,omitempty"`
170+
// Replicas for the lineage Deployment. Default 1.
171+
// +kubebuilder:default=1
172+
// +kubebuilder:validation:Minimum=1
173+
// +optional
174+
Replicas *int32 `json:"replicas,omitempty"`
143175
}
144176

145177
// MlflowConfig enables MLflow experiment tracking integration for Feast.
@@ -1090,6 +1122,7 @@ type ServiceHostnames struct {
10901122
Registry string `json:"registry,omitempty"`
10911123
RegistryRest string `json:"registryRest,omitempty"`
10921124
UI string `json:"ui,omitempty"`
1125+
Lineage string `json:"lineage,omitempty"`
10931126
}
10941127

10951128
// +kubebuilder:object:root=true

‎infra/feast-operator/api/v1/zz_generated.deepcopy.go‎

Lines changed: 40 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ metadata:
147147
}
148148
]
149149
capabilities: Basic Install
150-
createdAt: "2026-08-18T16:15:46Z"
150+
createdAt: "2026-08-19T16:50:46Z"
151151
operators.operatorframework.io/builder: operator-sdk-v1.41.0
152152
operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
153153
name: feast-operator.v0.65.0

0 commit comments

Comments
 (0)