Skip to content

Commit a79c77b

Browse files
Chen Zhilingfeast-ci-bot
Chen Zhiling
authored and
feast-ci-bot
committed
Add prometheus metrics to serving (#316)
* Add prometheus metrics to serving * Add subsystem * Use histogram instead of summary * Change timer to histogram timer * Fix configuration for serving chart for prometheus * Fix application.yaml reference in chart
1 parent 31aaed4 commit a79c77b

9 files changed

Lines changed: 140 additions & 9 deletions

File tree

‎infra/charts/feast/charts/feast-serving/templates/deployment.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@ metadata:
99
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
1010
release: {{ .Release.Name }}
1111
heritage: {{ .Release.Service }}
12+
annotations:
13+
{{- if .Values.prometheus.enabled }}
14+
{{ $config := index .Values "application.yaml" }}
15+
prometheus.io/path: /metrics
16+
prometheus.io/port: "{{ $config.server.port }}"
17+
prometheus.io/scrape: "true"
18+
{{- end }}
1219
spec:
1320
replicas: {{ .Values.replicaCount }}
1421
selector:

‎infra/charts/feast/charts/feast-serving/values.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ application.yaml:
6363
grpc:
6464
port: 6566
6565
enable-reflection: true
66+
server:
67+
port: 8080
6668
spring:
6769
main:
6870
web-application-type: none
@@ -177,6 +179,9 @@ ingress:
177179
# - host: chart-example.local
178180
# port: http
179181

182+
prometheus:
183+
enabled: true
184+
180185
resources: {}
181186
# We usually recommend not to specify default resources and to leave this as a conscious
182187
# choice for the user. This also increases chances charts run on environments with little

‎serving/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,30 @@
168168
<version>0.31.0</version>
169169
</dependency>
170170

171+
<!-- The client -->
172+
<dependency>
173+
<groupId>io.prometheus</groupId>
174+
<artifactId>simpleclient</artifactId>
175+
<version>0.8.0</version>
176+
</dependency>
177+
<!-- Hotspot JVM metrics-->
178+
<dependency>
179+
<groupId>io.prometheus</groupId>
180+
<artifactId>simpleclient_hotspot</artifactId>
181+
<version>0.8.0</version>
182+
</dependency>
183+
<!-- Exposition HTTPServer-->
184+
<dependency>
185+
<groupId>io.prometheus</groupId>
186+
<artifactId>simpleclient_servlet</artifactId>
187+
<version>0.8.0</version>
188+
</dependency>
189+
<dependency>
190+
<groupId>io.prometheus</groupId>
191+
<artifactId>simpleclient_spring_boot</artifactId>
192+
<version>0.8.0</version>
193+
</dependency>
194+
171195
<!-- Google Cloud -->
172196
<dependency>
173197
<groupId>com.google.cloud</groupId>

‎serving/src/main/java/feast/serving/configuration/InstrumentationConfig.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,29 @@
33
import feast.serving.FeastProperties;
44
import io.opentracing.Tracer;
55
import io.opentracing.noop.NoopTracerFactory;
6+
import io.prometheus.client.hotspot.DefaultExports;
7+
import io.prometheus.client.exporter.MetricsServlet;
68
import org.springframework.beans.factory.annotation.Autowired;
9+
import org.springframework.boot.web.servlet.ServletRegistrationBean;
710
import org.springframework.context.annotation.Bean;
811
import org.springframework.context.annotation.Configuration;
912

1013
@Configuration
1114
public class InstrumentationConfig {
15+
1216
private FeastProperties feastProperties;
1317

1418
@Autowired
1519
public InstrumentationConfig(FeastProperties feastProperties) {
1620
this.feastProperties = feastProperties;
1721
}
1822

23+
@Bean
24+
public ServletRegistrationBean servletRegistrationBean() {
25+
DefaultExports.initialize();
26+
return new ServletRegistrationBean(new MetricsServlet(), "/metrics");
27+
}
28+
1929
@Bean
2030
public Tracer tracer() {
2131
if (!feastProperties.getTracing().isEnabled()) {

‎serving/src/main/java/feast/serving/service/BigQueryServingService.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package feast.serving.service;
22

33
import static feast.serving.util.BigQueryUtil.getTimestampLimitQuery;
4+
import static feast.serving.util.Metrics.requestCount;
5+
import static feast.serving.util.Metrics.requestLatency;
46

57
import com.google.cloud.bigquery.BigQuery;
68
import com.google.cloud.bigquery.BigQueryException;
@@ -37,6 +39,7 @@
3739
import feast.serving.ServingAPIProto.JobType;
3840
import feast.serving.util.BigQueryUtil;
3941
import io.grpc.Status;
42+
import io.prometheus.client.Histogram.Timer;
4043
import java.io.IOException;
4144
import java.util.ArrayList;
4245
import java.util.List;
@@ -100,11 +103,13 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getF
100103
*/
101104
@Override
102105
public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) {
103-
106+
Timer getBatchFeaturesTimer = requestLatency.labels("getBatchFeatures").startTimer();
104107
List<FeatureSetSpec> featureSetSpecs =
105108
getFeaturesRequest.getFeatureSetsList().stream()
106-
.map(featureSet ->
107-
specService.getFeatureSet(featureSet.getName(), featureSet.getVersion())
109+
.map(featureSet -> {
110+
requestCount.labels(featureSet.getName()).inc();
111+
return specService.getFeatureSet(featureSet.getName(), featureSet.getVersion());
112+
}
108113
)
109114
.collect(Collectors.toList());
110115

@@ -233,6 +238,7 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat
233238
})
234239
.start();
235240

241+
getBatchFeaturesTimer.observeDuration();
236242
return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build();
237243
}
238244

‎serving/src/main/java/feast/serving/service/CachedSpecService.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import feast.core.StoreProto.Store.Subscription;
1717
import feast.serving.exception.SpecRetrievalException;
1818
import io.grpc.StatusRuntimeException;
19+
import io.prometheus.client.Gauge;
1920
import java.io.IOException;
2021
import java.nio.file.Files;
2122
import java.nio.file.Path;
@@ -40,6 +41,15 @@ public class CachedSpecService {
4041
private final LoadingCache<String, FeatureSetSpec> featureSetSpecCache;
4142
private Store store;
4243

44+
private static Gauge featureSetsCount = Gauge.build().name("feature_set_count")
45+
.subsystem("feast_serving")
46+
.help("number of feature sets served by this instance")
47+
.register();
48+
private static Gauge cacheLastUpdated = Gauge.build().name("cache_last_updated")
49+
.subsystem("feast_serving")
50+
.help("epoch time of the last time the cache was updated")
51+
.register();
52+
4353
public CachedSpecService(CoreSpecService coreService, Path configPath) {
4454
this.configPath = configPath;
4555
this.coreService = coreService;
@@ -102,6 +112,9 @@ public void populateCache() {
102112
this.store = updateStore(readConfig(configPath));
103113
Map<String, FeatureSetSpec> featureSetSpecMap = getFeatureSetSpecMap();
104114
featureSetSpecCache.putAll(featureSetSpecMap);
115+
116+
featureSetsCount.set(featureSetSpecCache.size());
117+
cacheLastUpdated.set(System.currentTimeMillis());
105118
}
106119

107120
public void scheduledPopulateCache() {

‎serving/src/main/java/feast/serving/service/RedisServingService.java‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@
1616

1717
package feast.serving.service;
1818

19+
import static feast.serving.util.Metrics.missingKeyCount;
20+
import static feast.serving.util.Metrics.requestLatency;
21+
import static feast.serving.util.Metrics.requestCount;
22+
import static feast.serving.util.Metrics.staleKeyCount;
23+
1924
import com.google.common.collect.Maps;
2025
import com.google.protobuf.AbstractMessageLite;
2126
import com.google.protobuf.Duration;
@@ -41,6 +46,7 @@
4146
import io.grpc.Status;
4247
import io.opentracing.Scope;
4348
import io.opentracing.Tracer;
49+
import io.prometheus.client.Histogram.Timer;
4450
import java.util.List;
4551
import java.util.Map;
4652
import java.util.stream.Collectors;
@@ -61,7 +67,9 @@ public RedisServingService(JedisPool jedisPool, CachedSpecService specService, T
6167
this.tracer = tracer;
6268
}
6369

64-
/** {@inheritDoc} */
70+
/**
71+
* {@inheritDoc}
72+
*/
6573
@Override
6674
public GetFeastServingInfoResponse getFeastServingInfo(
6775
GetFeastServingInfoRequest getFeastServingInfoRequest) {
@@ -70,10 +78,13 @@ public GetFeastServingInfoResponse getFeastServingInfo(
7078
.build();
7179
}
7280

73-
/** {@inheritDoc} */
81+
/**
82+
* {@inheritDoc}
83+
*/
7484
@Override
7585
public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) {
7686
try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) {
87+
Timer getOnlineFeaturesTimer = requestLatency.labels("getOnlineFeatures").startTimer();
7788
GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder =
7889
GetOnlineFeaturesResponse.newBuilder();
7990

@@ -114,6 +125,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ
114125
featureValuesMap.values().stream()
115126
.map(m -> FieldValues.newBuilder().putAllFields(m).build())
116127
.collect(Collectors.toList());
128+
getOnlineFeaturesTimer.observeDuration();
117129
return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build();
118130
}
119131
}
@@ -166,9 +178,11 @@ private RedisKey makeRedisKey(
166178
for (int i = 0; i < featureSetEntityNames.size(); i++) {
167179
String entityName = featureSetEntityNames.get(i);
168180

169-
if (!fieldsMap.containsKey(entityName)){
181+
if (!fieldsMap.containsKey(entityName)) {
170182
throw Status.INVALID_ARGUMENT
171-
.withDescription(String.format("Entity row fields \"%s\" does not contain required entity field \"%s\"", fieldsMap.keySet().toString(), entityName))
183+
.withDescription(String
184+
.format("Entity row fields \"%s\" does not contain required entity field \"%s\"",
185+
fieldsMap.keySet().toString(), entityName))
172186
.asRuntimeException();
173187
}
174188

@@ -186,33 +200,45 @@ private void sendAndProcessMultiGet(
186200
throws InvalidProtocolBufferException {
187201

188202
List<byte[]> jedisResps = sendMultiGet(redisKeys);
189-
203+
Timer processResponseTimer = requestLatency.labels("processResponse")
204+
.startTimer();
190205
try (Scope scope = tracer.buildSpan("Redis-processResponse").startActive(true)) {
191206
String featureSetId =
192207
String.format("%s:%d", featureSetRequest.getName(), featureSetRequest.getVersion());
208+
193209
Map<String, Value> nullValues =
194210
featureSetRequest.getFeatureNamesList().stream()
195211
.collect(
196212
Collectors.toMap(
197213
name -> featureSetId + ":" + name, name -> Value.newBuilder().build()));
214+
198215
for (int i = 0; i < jedisResps.size(); i++) {
199216
EntityRow entityRow = entityRows.get(i);
200217
Map<String, Value> featureValues = featureValuesMap.get(entityRow);
218+
201219
byte[] jedisResponse = jedisResps.get(i);
202220
if (jedisResponse == null) {
221+
missingKeyCount.labels(featureSetRequest.getName()).inc();
203222
featureValues.putAll(nullValues);
204223
continue;
205224
}
225+
206226
FeatureRow featureRow = FeatureRow.parseFrom(jedisResponse);
227+
207228
boolean stale = isStale(featureSetRequest, entityRow, featureRow);
208229
if (stale) {
230+
staleKeyCount.labels(featureSetRequest.getName()).inc();
209231
featureValues.putAll(nullValues);
210232
continue;
211233
}
234+
235+
requestCount.labels(featureSetRequest.getName()).inc();
212236
featureRow.getFieldsList().stream()
213237
.filter(f -> featureSetRequest.getFeatureNamesList().contains(f.getName()))
214238
.forEach(f -> featureValues.put(featureSetId + ":" + f.getName(), f.getValue()));
215239
}
240+
} finally {
241+
processResponseTimer.observeDuration();
216242
}
217243
}
218244

@@ -237,6 +263,7 @@ private boolean isStale(
237263
*/
238264
private List<byte[]> sendMultiGet(List<RedisKey> keys) {
239265
try (Scope scope = tracer.buildSpan("Redis-sendMultiGet").startActive(true)) {
266+
Timer sendMultiGetTimer = requestLatency.labels("sendMultiGet").startTimer();
240267
try (Jedis jedis = jedisPool.getResource()) {
241268
byte[][] binaryKeys =
242269
keys.stream()
@@ -249,6 +276,8 @@ private List<byte[]> sendMultiGet(List<RedisKey> keys) {
249276
.withDescription("Unable to retrieve feature from Redis")
250277
.withCause(e)
251278
.asRuntimeException();
279+
} finally {
280+
sendMultiGetTimer.observeDuration();
252281
}
253282
}
254283
}

‎serving/src/main/java/feast/serving/util/Metrics.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package feast.serving.util;
2+
3+
import io.prometheus.client.Counter;
4+
import io.prometheus.client.Histogram;
5+
import io.prometheus.client.Summary;
6+
7+
public class Metrics {
8+
9+
public static final Histogram requestLatency = Histogram.build()
10+
.buckets(2, 4, 6, 8, 10, 15, 20, 25, 30, 35, 50)
11+
.name("request_latency_ms")
12+
.subsystem("feast_serving")
13+
.help("Request latency in milliseconds.")
14+
.labelNames("method")
15+
.register();
16+
17+
public static final Counter requestCount = Counter.build()
18+
.name("request_feature_count")
19+
.subsystem("feast_serving")
20+
.help("number of feature rows requested")
21+
.labelNames("feature_set_name")
22+
.register();
23+
24+
public static final Counter missingKeyCount = Counter.build()
25+
.name("missing_feature_count")
26+
.subsystem("feast_serving")
27+
.help("number requested feature rows that were not found")
28+
.labelNames("feature_set_name")
29+
.register();
30+
31+
public static final Counter staleKeyCount = Counter.build()
32+
.name("stale_feature_count")
33+
.subsystem("feast_serving")
34+
.help("number requested feature rows that were stale")
35+
.labelNames("feature_set_name")
36+
.register();
37+
}

‎serving/src/main/resources/application.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,6 @@ grpc:
5555

5656
server:
5757
# The port number on which the Tomcat webserver that serves REST API endpoints should listen
58-
# It is set by default to 8080 so it does not conflict with Tomcat webserver on Feast Core
58+
# It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core
5959
# if both Feast Core and Serving are running on the same machine
6060
port: ${SERVER_PORT:8081}

0 commit comments

Comments
 (0)