Skip to content

Commit 6d127e3

Browse files
mutianfmutianf
committed
feat: migrate builtin metrics to OTEl
1 parent e2b7e58 commit 6d127e3

File tree

12 files changed

+915
-193
lines changed

12 files changed

+915
-193
lines changed

‎google-cloud-bigtable/pom.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,15 @@
336336
<groupId>io.opentelemetry</groupId>
337337
<artifactId>opentelemetry-api</artifactId>
338338
</dependency>
339+
<dependency>
340+
<groupId>io.opentelemetry</groupId>
341+
<artifactId>opentelemetry-sdk</artifactId>
342+
</dependency>
343+
<dependency>
344+
<groupId>io.opentelemetry</groupId>
345+
<artifactId>opentelemetry-sdk-testing</artifactId>
346+
<scope>test</scope>
347+
</dependency>
339348
<dependency>
340349
<groupId>io.opentelemetry</groupId>
341350
<artifactId>opentelemetry-sdk-metrics</artifactId>

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,10 @@
2525
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2626
import com.google.api.gax.rpc.UnaryCallSettings;
2727
import com.google.auth.Credentials;
28-
import com.google.auth.oauth2.GoogleCredentials;
2928
import com.google.cloud.bigtable.data.v2.models.Query;
3029
import com.google.cloud.bigtable.data.v2.models.Row;
3130
import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
3231
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
33-
import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter;
34-
import com.google.cloud.bigtable.stats.BuiltinViews;
3532
import com.google.common.base.MoreObjects;
3633
import com.google.common.base.Strings;
3734
import io.grpc.ManagedChannelBuilder;
@@ -197,23 +194,26 @@ public static void enableGfeOpenCensusStats() {
197194
com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews();
198195
}
199196

200-
/** Register built in metrics. */
197+
/**
198+
* Register built in metrics.
199+
*
200+
* @deprecated Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)}
201+
* to enable or disable built-in metrics.
202+
*/
203+
@Deprecated
201204
public static void enableBuiltinMetrics() throws IOException {
202-
if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
203-
BuiltinViews.registerBigtableBuiltinViews();
204-
BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault());
205-
}
205+
BUILTIN_METRICS_REGISTERED.compareAndSet(false, true);
206206
}
207207

208208
/**
209209
* Register built in metrics with credentials. The credentials need to have metric write access
210210
* for all the projects you're publishing to.
211+
*
212+
* @deprecated Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)}
213+
* to enable or disable built-in metrics.
211214
*/
212215
public static void enableBuiltinMetrics(Credentials credentials) throws IOException {
213-
if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
214-
BuiltinViews.registerBigtableBuiltinViews();
215-
BigtableStackdriverStatsExporter.register(credentials);
216-
}
216+
BUILTIN_METRICS_REGISTERED.compareAndSet(false, true);
217217
}
218218

219219
/** Returns the target project id. */
@@ -278,6 +278,11 @@ public boolean isBulkMutationFlowControlEnabled() {
278278
return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
279279
}
280280

281+
/** Gets if built-in metrics is enabled. */
282+
public boolean isBuiltinMetricsEnabled() {
283+
return stubSettings.isBuiltinMetricsEnabled();
284+
}
285+
281286
/** Returns the underlying RPC settings. */
282287
public EnhancedBigtableStubSettings getStubSettings() {
283288
return stubSettings;
@@ -527,6 +532,17 @@ public boolean isBulkMutationFlowControlEnabled() {
527532
return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
528533
}
529534

535+
/** Set if built-in metrics will be enabled. */
536+
public Builder setBuiltinMetricsEnabled(boolean enable) {
537+
stubSettings.setBuiltinMetricsEnabled(enable);
538+
return this;
539+
}
540+
541+
/** Gets if built-in metrics is enabled. */
542+
public boolean isBuiltinMetricsEnabled() {
543+
return stubSettings.isBuiltinMetricsEnabled();
544+
}
545+
530546
/**
531547
* Returns the underlying settings for making RPC calls. The settings should be changed with
532548
* care.

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -220,12 +220,6 @@ public static EnhancedBigtableStubSettings finalizeSettings(
220220
RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID,
221221
TagValue.create(settings.getAppProfileId()))
222222
.build();
223-
ImmutableMap<String, String> builtinAttributes =
224-
ImmutableMap.<String, String>builder()
225-
.put("project_id", settings.getProjectId())
226-
.put("instance", settings.getInstanceId())
227-
.put("app_profile", settings.getAppProfileId())
228-
.build();
229223
// Inject Opencensus instrumentation
230224
builder.setTracerFactory(
231225
new CompositeTracerFactory(
@@ -250,7 +244,7 @@ public static EnhancedBigtableStubSettings finalizeSettings(
250244
.build()),
251245
// Add OpenCensus Metrics
252246
MetricsTracerFactory.create(tagger, stats, attributes),
253-
BuiltinMetricsTracerFactory.create(builtinAttributes),
247+
BuiltinMetricsTracerFactory.create(settings),
254248
// Add user configured tracer
255249
settings.getTracerFactory())));
256250
return builder.build();

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ public class EnhancedBigtableStubSettings extends StubSettings<EnhancedBigtableS
228228

229229
private final FeatureFlags featureFlags;
230230

231+
private final boolean isBuiltinMetricsEnabled;
232+
231233
private EnhancedBigtableStubSettings(Builder builder) {
232234
super(builder);
233235

@@ -252,6 +254,7 @@ private EnhancedBigtableStubSettings(Builder builder) {
252254
isRefreshingChannel = builder.isRefreshingChannel;
253255
primedTableIds = builder.primedTableIds;
254256
jwtAudienceMapping = builder.jwtAudienceMapping;
257+
isBuiltinMetricsEnabled = builder.isBuiltinMetricsEnabled;
255258

256259
// Per method settings.
257260
readRowsSettings = builder.readRowsSettings.build();
@@ -313,6 +316,10 @@ public Map<String, String> getJwtAudienceMapping() {
313316
return jwtAudienceMapping;
314317
}
315318

319+
public boolean isBuiltinMetricsEnabled() {
320+
return isBuiltinMetricsEnabled;
321+
}
322+
316323
/** Returns a builder for the default ChannelProvider for this service. */
317324
public static InstantiatingGrpcChannelProvider.Builder defaultGrpcTransportProviderBuilder() {
318325
return BigtableStubSettings.defaultGrpcTransportProviderBuilder()
@@ -613,6 +620,8 @@ public static class Builder extends StubSettings.Builder<EnhancedBigtableStubSet
613620

614621
private FeatureFlags.Builder featureFlags;
615622

623+
private boolean isBuiltinMetricsEnabled;
624+
616625
/**
617626
* Initializes a new Builder with sane defaults for all settings.
618627
*
@@ -624,6 +633,7 @@ public static class Builder extends StubSettings.Builder<EnhancedBigtableStubSet
624633
private Builder() {
625634
this.appProfileId = SERVER_DEFAULT_APP_PROFILE_ID;
626635
this.isRefreshingChannel = true;
636+
this.isBuiltinMetricsEnabled = true;
627637
primedTableIds = ImmutableList.of();
628638
jwtAudienceMapping = DEFAULT_JWT_AUDIENCE_MAPPING;
629639
setCredentialsProvider(defaultCredentialsProviderBuilder().build());
@@ -886,6 +896,17 @@ public Builder setJwtAudienceMapping(Map<String, String> jwtAudienceMapping) {
886896
return this;
887897
}
888898

899+
/** Returns true if builtin metrics is enabled. */
900+
public boolean isBuiltinMetricsEnabled() {
901+
return isBuiltinMetricsEnabled;
902+
}
903+
904+
/** Set if builtin metrics will be enabled. */
905+
public Builder setBuiltinMetricsEnabled(boolean enable) {
906+
this.isBuiltinMetricsEnabled = enable;
907+
return this;
908+
}
909+
889910
@InternalApi("Used for internal testing")
890911
public Map<String, String> getJwtAudienceMapping() {
891912
return jwtAudienceMapping;
@@ -1030,6 +1051,7 @@ public String toString() {
10301051
generateInitialChangeStreamPartitionsSettings)
10311052
.add("readChangeStreamSettings", readChangeStreamSettings)
10321053
.add("pingAndWarmSettings", pingAndWarmSettings)
1054+
.add("isBuiltinMetricsEnabled", isBuiltinMetricsEnabled)
10331055
.add("parent", super.toString())
10341056
.toString();
10351057
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ class BigtableExporterUtils {
6767

6868
private static final Logger logger = Logger.getLogger(BigtableExporterUtils.class.getName());
6969

70+
private static final String METRIC_PREFIX = "bigtable.googleapis.com/internal/client/";
71+
7072
static String getDefaultTaskValue() {
7173
// Something like '<pid>@<hostname>'
7274
final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
@@ -117,7 +119,7 @@ static TimeSeries convertPointToTimeSeries(
117119
.setMetricKind(convertMetricKind(metricData))
118120
.setMetric(
119121
Metric.newBuilder()
120-
.setType(metricData.getName())
122+
.setType(METRIC_PREFIX + metricData.getName())
121123
.putAllLabels(metricLabels.build())
122124
.build())
123125
.setValueType(convertValueType(metricData.getType()));

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableMetricsRecorder.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.stub.metrics;
17+
18+
import io.opentelemetry.api.common.Attributes;
19+
20+
public class BigtableMetricsRecorder {
21+
22+
void recordOperationLatencies(long value, Attributes attributes) {}
23+
24+
void recordAttemptLatencies(long value, Attributes attributes) {}
25+
26+
void recordFirstResponseLatencies(long value, Attributes attributes) {}
27+
28+
void recordRetryCount(long value, Attributes attributes) {}
29+
30+
void recordServerLatencies(long value, Attributes attributes) {}
31+
32+
void recordConnectivityErrorCount(long value, Attributes attributes) {}
33+
34+
void recordApplicationBlockingLatencies(long value, Attributes attributes) {}
35+
36+
void recordClientBlockingLatencies(long value, Attributes attributes) {}
37+
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinInMetricsRecorder.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.stub.metrics;
17+
18+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.APPLICATION_BLOCKING_LATENCIES_NAME;
19+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.ATTEMPT_LATENCIES_NAME;
20+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CLIENT_BLOCKING_LATENCIES_NAME;
21+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CONNECTIVITY_ERROR_COUNT_NAME;
22+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.FIRST_RESPONSE_LATENCIES_NAME;
23+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.OPERATION_LATENCIES_NAME;
24+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.RETRY_COUNT_NAME;
25+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.SERVER_LATENCIES_NAME;
26+
27+
import io.opentelemetry.api.common.Attributes;
28+
import io.opentelemetry.api.metrics.LongCounter;
29+
import io.opentelemetry.api.metrics.LongHistogram;
30+
import io.opentelemetry.api.metrics.Meter;
31+
32+
class BuiltinInMetricsRecorder extends BigtableMetricsRecorder {
33+
34+
private static final String MILLISECOND = "ms";
35+
private static final String COUNT = "1";
36+
37+
private final LongHistogram operationLatencies;
38+
private final LongHistogram attemptLatencies;
39+
private final LongHistogram serverLatencies;
40+
private final LongHistogram firstResponseLatencies;
41+
private final LongHistogram clientBlockingLatencies;
42+
private final LongHistogram applicationBlockingLatencies;
43+
private final LongCounter connectivityErrorCount;
44+
private final LongCounter retryCount;
45+
46+
BuiltinInMetricsRecorder(Meter meter) {
47+
operationLatencies =
48+
meter
49+
.histogramBuilder(OPERATION_LATENCIES_NAME)
50+
.ofLongs()
51+
.setDescription(
52+
"Total time until final operation success or failure, including retries and backoff.")
53+
.setUnit(MILLISECOND)
54+
.build();
55+
attemptLatencies =
56+
meter
57+
.histogramBuilder(ATTEMPT_LATENCIES_NAME)
58+
.ofLongs()
59+
.setDescription("Client observed latency per RPC attempt.")
60+
.setUnit(MILLISECOND)
61+
.build();
62+
serverLatencies =
63+
meter
64+
.histogramBuilder(SERVER_LATENCIES_NAME)
65+
.ofLongs()
66+
.setDescription(
67+
"The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.")
68+
.setUnit(MILLISECOND)
69+
.build();
70+
firstResponseLatencies =
71+
meter
72+
.histogramBuilder(FIRST_RESPONSE_LATENCIES_NAME)
73+
.ofLongs()
74+
.setDescription(
75+
"Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.")
76+
.setUnit(MILLISECOND)
77+
.build();
78+
clientBlockingLatencies =
79+
meter
80+
.histogramBuilder(CLIENT_BLOCKING_LATENCIES_NAME)
81+
.ofLongs()
82+
.setDescription(
83+
"The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.")
84+
.setUnit(MILLISECOND)
85+
.build();
86+
applicationBlockingLatencies =
87+
meter
88+
.histogramBuilder(APPLICATION_BLOCKING_LATENCIES_NAME)
89+
.ofLongs()
90+
.setDescription(
91+
"The latency of the client application consuming available response data.")
92+
.setUnit(MILLISECOND)
93+
.build();
94+
connectivityErrorCount =
95+
meter
96+
.counterBuilder(CONNECTIVITY_ERROR_COUNT_NAME)
97+
.setDescription(
98+
"Number of requests that failed to reach the Google datacenter. (Requests without google response headers")
99+
.setUnit(COUNT)
100+
.build();
101+
retryCount =
102+
meter
103+
.counterBuilder(RETRY_COUNT_NAME)
104+
.setDescription("The number of additional RPCs sent after the initial attempt.")
105+
.setUnit(COUNT)
106+
.build();
107+
}
108+
109+
@Override
110+
void recordOperationLatencies(long value, Attributes attributes) {
111+
operationLatencies.record(value, attributes);
112+
}
113+
114+
@Override
115+
void recordAttemptLatencies(long value, Attributes attributes) {
116+
attemptLatencies.record(value, attributes);
117+
}
118+
119+
@Override
120+
void recordFirstResponseLatencies(long value, Attributes attributes) {
121+
firstResponseLatencies.record(value, attributes);
122+
}
123+
124+
@Override
125+
void recordRetryCount(long value, Attributes attributes) {
126+
retryCount.add(value, attributes);
127+
}
128+
129+
@Override
130+
void recordServerLatencies(long value, Attributes attributes) {
131+
serverLatencies.record(value, attributes);
132+
}
133+
134+
@Override
135+
void recordConnectivityErrorCount(long value, Attributes attributes) {
136+
connectivityErrorCount.add(value, attributes);
137+
}
138+
139+
@Override
140+
void recordApplicationBlockingLatencies(long value, Attributes attributes) {
141+
applicationBlockingLatencies.record(value, attributes);
142+
}
143+
144+
@Override
145+
void recordClientBlockingLatencies(long value, Attributes attributes) {
146+
clientBlockingLatencies.record(value, attributes);
147+
}
148+
}

0 commit comments

Comments
 (0)