Skip to content

Commit 2209c62

Browse files
committed
feat: Add ServiceMonitor auto-generation for Prometheus discovery
Signed-off-by: ntkathole <nikhilkathole2683@gmail.com>
1 parent 18ede88 commit 2209c62

File tree

8 files changed

+328
-3
lines changed

8 files changed

+328
-3
lines changed

.secrets.baseline

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,7 +1156,7 @@
11561156
"filename": "infra/feast-operator/internal/controller/services/services.go",
11571157
"hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1",
11581158
"is_verified": false,
1159-
"line_number": 176
1159+
"line_number": 179
11601160
}
11611161
],
11621162
"infra/feast-operator/internal/controller/services/tls_test.go": [
@@ -1539,5 +1539,5 @@
15391539
}
15401540
]
15411541
},
1542-
"generated_at": "2026-03-14T16:01:28Z"
1542+
"generated_at": "2026-03-18T13:51:43Z"
15431543
}

infra/feast-operator/config/rbac/role.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,17 @@ rules:
101101
- get
102102
- patch
103103
- update
104+
- apiGroups:
105+
- monitoring.coreos.com
106+
resources:
107+
- servicemonitors
108+
verbs:
109+
- create
110+
- delete
111+
- get
112+
- list
113+
- update
114+
- watch
104115
- apiGroups:
105116
- policy
106117
resources:

infra/feast-operator/internal/controller/featurestore_controller.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ import (
3030
apierrors "k8s.io/apimachinery/pkg/api/errors"
3131
apimeta "k8s.io/apimachinery/pkg/api/meta"
3232
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3334
"k8s.io/apimachinery/pkg/runtime"
35+
"k8s.io/apimachinery/pkg/runtime/schema"
3436
"k8s.io/apimachinery/pkg/types"
3537
ctrl "sigs.k8s.io/controller-runtime"
3638
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -69,6 +71,7 @@ type FeatureStoreReconciler struct {
6971
// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete
7072
// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
7173
// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
74+
// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;delete
7275

7376
// Reconcile is part of the main kubernetes reconciliation loop which aims to
7477
// move the current state of the cluster closer to the desired state.
@@ -244,6 +247,15 @@ func (r *FeatureStoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
244247
if services.IsOpenShift() {
245248
bldr = bldr.Owns(&routev1.Route{})
246249
}
250+
if services.HasServiceMonitorCRD() {
251+
sm := &unstructured.Unstructured{}
252+
sm.SetGroupVersionKind(schema.GroupVersionKind{
253+
Group: "monitoring.coreos.com",
254+
Version: "v1",
255+
Kind: "ServiceMonitor",
256+
})
257+
bldr = bldr.Owns(sm)
258+
}
247259

248260
return bldr.Complete(r)
249261

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
Copyright 2026 Feast Community.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package services
18+
19+
import (
20+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
21+
"k8s.io/apimachinery/pkg/runtime/schema"
22+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
23+
"sigs.k8s.io/controller-runtime/pkg/log"
24+
)
25+
26+
var serviceMonitorGVK = schema.GroupVersionKind{
27+
Group: "monitoring.coreos.com",
28+
Version: "v1",
29+
Kind: "ServiceMonitor",
30+
}
31+
32+
// createOrDeleteServiceMonitor reconciles the ServiceMonitor for the
33+
// FeatureStore's online store metrics endpoint. When the Prometheus Operator
34+
// CRD is not present in the cluster, this is a no-op. When metrics are enabled
35+
// on the online store, a ServiceMonitor is created; otherwise any existing
36+
// ServiceMonitor is deleted.
37+
func (feast *FeastServices) createOrDeleteServiceMonitor() error {
38+
if !hasServiceMonitorCRD {
39+
return nil
40+
}
41+
42+
if feast.isOnlineStore() && feast.isMetricsEnabled(OnlineFeastType) {
43+
return feast.createServiceMonitor()
44+
}
45+
46+
return feast.deleteServiceMonitor()
47+
}
48+
49+
func (feast *FeastServices) createServiceMonitor() error {
50+
logger := log.FromContext(feast.Handler.Context)
51+
sm := feast.initServiceMonitor()
52+
if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sm, controllerutil.MutateFn(func() error {
53+
return feast.setServiceMonitor(sm)
54+
})); err != nil {
55+
return err
56+
} else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated {
57+
logger.Info("Successfully reconciled", "ServiceMonitor", sm.GetName(), "operation", op)
58+
}
59+
return nil
60+
}
61+
62+
func (feast *FeastServices) deleteServiceMonitor() error {
63+
sm := feast.initServiceMonitor()
64+
return feast.Handler.DeleteOwnedFeastObj(sm)
65+
}
66+
67+
func (feast *FeastServices) initServiceMonitor() *unstructured.Unstructured {
68+
sm := &unstructured.Unstructured{}
69+
sm.SetGroupVersionKind(serviceMonitorGVK)
70+
sm.SetName(feast.GetFeastServiceName(OnlineFeastType))
71+
sm.SetNamespace(feast.Handler.FeatureStore.Namespace)
72+
return sm
73+
}
74+
75+
func (feast *FeastServices) setServiceMonitor(sm *unstructured.Unstructured) error {
76+
cr := feast.Handler.FeatureStore
77+
78+
sm.SetLabels(feast.getFeastTypeLabels(OnlineFeastType))
79+
80+
sm.Object["spec"] = map[string]interface{}{
81+
"endpoints": []interface{}{
82+
map[string]interface{}{
83+
"port": "metrics",
84+
"path": "/metrics",
85+
},
86+
},
87+
"selector": map[string]interface{}{
88+
"matchLabels": map[string]interface{}{
89+
NameLabelKey: cr.Name,
90+
ServiceTypeLabelKey: string(OnlineFeastType),
91+
},
92+
},
93+
}
94+
95+
return controllerutil.SetControllerReference(cr, sm, feast.Handler.Scheme)
96+
}
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/*
2+
Copyright 2026 Feast Community.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package services
18+
19+
import (
20+
"context"
21+
22+
feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1"
23+
"github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler"
24+
. "github.com/onsi/ginkgo/v2"
25+
. "github.com/onsi/gomega"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
28+
"k8s.io/apimachinery/pkg/types"
29+
"k8s.io/utils/ptr"
30+
)
31+
32+
var _ = Describe("ServiceMonitor", func() {
33+
var (
34+
featureStore *feastdevv1.FeatureStore
35+
feast *FeastServices
36+
typeNamespacedName types.NamespacedName
37+
ctx context.Context
38+
)
39+
40+
BeforeEach(func() {
41+
ctx = context.Background()
42+
typeNamespacedName = types.NamespacedName{
43+
Name: "sm-test-fs",
44+
Namespace: "default",
45+
}
46+
47+
featureStore = &feastdevv1.FeatureStore{
48+
ObjectMeta: metav1.ObjectMeta{
49+
Name: typeNamespacedName.Name,
50+
Namespace: typeNamespacedName.Namespace,
51+
},
52+
Spec: feastdevv1.FeatureStoreSpec{
53+
FeastProject: "smtestproject",
54+
Services: &feastdevv1.FeatureStoreServices{
55+
OnlineStore: &feastdevv1.OnlineStore{
56+
Server: &feastdevv1.ServerConfigs{
57+
ContainerConfigs: feastdevv1.ContainerConfigs{
58+
DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{
59+
Image: ptr.To("test-image"),
60+
},
61+
},
62+
},
63+
},
64+
Registry: &feastdevv1.Registry{
65+
Local: &feastdevv1.LocalRegistryConfig{
66+
Server: &feastdevv1.RegistryServerConfigs{
67+
ServerConfigs: feastdevv1.ServerConfigs{
68+
ContainerConfigs: feastdevv1.ContainerConfigs{
69+
DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{
70+
Image: ptr.To("test-image"),
71+
},
72+
},
73+
},
74+
GRPC: ptr.To(true),
75+
},
76+
},
77+
},
78+
},
79+
},
80+
}
81+
82+
Expect(k8sClient.Create(ctx, featureStore)).To(Succeed())
83+
84+
feast = &FeastServices{
85+
Handler: handler.FeastHandler{
86+
Client: k8sClient,
87+
Context: ctx,
88+
Scheme: k8sClient.Scheme(),
89+
FeatureStore: featureStore,
90+
},
91+
}
92+
93+
Expect(feast.ApplyDefaults()).To(Succeed())
94+
applySpecToStatus(featureStore)
95+
feast.refreshFeatureStore(ctx, typeNamespacedName)
96+
})
97+
98+
AfterEach(func() {
99+
testSetHasServiceMonitorCRD(false)
100+
Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed())
101+
})
102+
103+
Describe("initServiceMonitor", func() {
104+
It("should create an unstructured ServiceMonitor with correct GVK and name", func() {
105+
sm := feast.initServiceMonitor()
106+
Expect(sm).NotTo(BeNil())
107+
Expect(sm.GetKind()).To(Equal("ServiceMonitor"))
108+
Expect(sm.GetAPIVersion()).To(Equal("monitoring.coreos.com/v1"))
109+
Expect(sm.GetName()).To(Equal(feast.GetFeastServiceName(OnlineFeastType)))
110+
Expect(sm.GetNamespace()).To(Equal(featureStore.Namespace))
111+
})
112+
})
113+
114+
Describe("setServiceMonitor", func() {
115+
It("should populate the ServiceMonitor spec with correct selector and endpoint", func() {
116+
sm := feast.initServiceMonitor()
117+
Expect(feast.setServiceMonitor(sm)).To(Succeed())
118+
119+
labels := sm.GetLabels()
120+
Expect(labels).To(HaveKeyWithValue(NameLabelKey, featureStore.Name))
121+
Expect(labels).To(HaveKeyWithValue(ServiceTypeLabelKey, string(OnlineFeastType)))
122+
123+
spec, ok := sm.Object["spec"].(map[string]interface{})
124+
Expect(ok).To(BeTrue())
125+
126+
endpoints, ok := spec["endpoints"].([]interface{})
127+
Expect(ok).To(BeTrue())
128+
Expect(endpoints).To(HaveLen(1))
129+
ep := endpoints[0].(map[string]interface{})
130+
Expect(ep["port"]).To(Equal("metrics"))
131+
Expect(ep["path"]).To(Equal("/metrics"))
132+
133+
selector, ok := spec["selector"].(map[string]interface{})
134+
Expect(ok).To(BeTrue())
135+
matchLabels := selector["matchLabels"].(map[string]interface{})
136+
Expect(matchLabels[NameLabelKey]).To(Equal(featureStore.Name))
137+
Expect(matchLabels[ServiceTypeLabelKey]).To(Equal(string(OnlineFeastType)))
138+
139+
ownerRefs := sm.GetOwnerReferences()
140+
Expect(ownerRefs).To(HaveLen(1))
141+
Expect(ownerRefs[0].Name).To(Equal(featureStore.Name))
142+
Expect(*ownerRefs[0].Controller).To(BeTrue())
143+
})
144+
})
145+
146+
Describe("createOrDeleteServiceMonitor", func() {
147+
It("should be a no-op when ServiceMonitor CRD is not available", func() {
148+
testSetHasServiceMonitorCRD(false)
149+
Expect(feast.createOrDeleteServiceMonitor()).To(Succeed())
150+
})
151+
152+
It("should not error when metrics is not enabled and CRD is unavailable", func() {
153+
testSetHasServiceMonitorCRD(false)
154+
featureStore.Status.Applied.Services.OnlineStore.Server.Metrics = ptr.To(false)
155+
Expect(feast.createOrDeleteServiceMonitor()).To(Succeed())
156+
})
157+
})
158+
159+
Describe("HasServiceMonitorCRD", func() {
160+
It("should return false by default", func() {
161+
testSetHasServiceMonitorCRD(false)
162+
Expect(HasServiceMonitorCRD()).To(BeFalse())
163+
})
164+
165+
It("should return true when set", func() {
166+
testSetHasServiceMonitorCRD(true)
167+
Expect(HasServiceMonitorCRD()).To(BeTrue())
168+
})
169+
})
170+
})
171+
172+
func verifyServiceMonitorUnstructured(sm *unstructured.Unstructured, featureStoreName, namespace string) {
173+
Expect(sm.GetKind()).To(Equal("ServiceMonitor"))
174+
Expect(sm.GetAPIVersion()).To(Equal("monitoring.coreos.com/v1"))
175+
Expect(sm.GetNamespace()).To(Equal(namespace))
176+
177+
labels := sm.GetLabels()
178+
Expect(labels).To(HaveKeyWithValue(NameLabelKey, featureStoreName))
179+
Expect(labels).To(HaveKeyWithValue(ServiceTypeLabelKey, string(OnlineFeastType)))
180+
181+
spec, ok := sm.Object["spec"].(map[string]interface{})
182+
Expect(ok).To(BeTrue())
183+
184+
endpoints, ok := spec["endpoints"].([]interface{})
185+
Expect(ok).To(BeTrue())
186+
Expect(endpoints).To(HaveLen(1))
187+
ep := endpoints[0].(map[string]interface{})
188+
Expect(ep["port"]).To(Equal("metrics"))
189+
Expect(ep["path"]).To(Equal("/metrics"))
190+
}

infra/feast-operator/internal/controller/services/services.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ func (feast *FeastServices) Deploy() error {
9696
if err := feast.deployCronJob(); err != nil {
9797
return err
9898
}
99+
if err := feast.createOrDeleteServiceMonitor(); err != nil {
100+
return err
101+
}
99102

100103
return nil
101104
}

infra/feast-operator/internal/controller/services/suite_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,7 @@ var _ = AfterSuite(func() {
8888
func testSetIsOpenShift() {
8989
isOpenShift = true
9090
}
91+
92+
func testSetHasServiceMonitorCRD(val bool) {
93+
hasServiceMonitorCRD = val
94+
}

0 commit comments

Comments
 (0)