From d308063c48c7b999b0f8c8887abb017f9f122659 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 19:33:24 -0400 Subject: [PATCH 1/9] Adding configs and metrics creation --- .../java/datadog/trace/bootstrap/Agent.java | 33 ++ .../dd-java-agent/reflect-config.json | 7 + .../shim/metrics/JvmOtlpRuntimeMetrics.java | 291 ++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 5288f92dbe3..1fa98a07ac6 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -26,6 +26,7 @@ import datadog.instrument.utils.ClassLoaderValue; import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.Platform; import datadog.trace.api.WithGlobalTracer; import datadog.trace.api.appsec.AppSecEventTracker; @@ -846,6 +847,17 @@ private static synchronized void installDatadogTracer( initTelemetry.onFatalError(ex); } + // Register JVM runtime metric callbacks against the OtelMeterProvider after + // CoreTracer has started OtlpMetricsService. Skip when OTEL_METRICS_EXPORTER=none + // since there's no exporter to collect against. Done here (not in the delayed + // startJmx) so callbacks are in place before the exporter's first flush. + Config cfg = Config.get(); + if (cfg.isRuntimeMetricsEnabled() + && InstrumenterConfig.get().isMetricsOtelEnabled() + && cfg.isMetricsOtlpExporterEnabled()) { + startOtlpRuntimeMetrics(); + } + StaticEventLogger.end("GlobalTracer"); } @@ -989,6 +1001,27 @@ private static synchronized void initializeJmxSystemAccessProvider( } } + /** + * Registers OTLP runtime metric callbacks (JVM heap, CPU, threads, classes, etc.) with the + * agent's OtelMeterProvider. The periodic OTLP exporter started by CoreTracer then collects and + * exports them — this is the same pattern Node and .NET use to start their runtime metrics + * unconditionally during tracer init, independent of any app-side OTel API usage. + */ + private static synchronized void startOtlpRuntimeMetrics() { + final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class jvmOtlpClass = + AGENT_CLASSLOADER.loadClass("datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics"); + final Method startMethod = jvmOtlpClass.getMethod("start"); + startMethod.invoke(null); + } catch (final Throwable ex) { + log.error("Throwable thrown while starting OTLP runtime metrics", ex); + } finally { + safelySetContextClassLoader(contextLoader); + } + } + private static synchronized void startJmxFetch() { final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); try { diff --git a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json index 3225d6dd59f..2865cfad830 100644 --- a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json +++ b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json @@ -218,5 +218,12 @@ "methods": [ {"name": "", "parameterTypes": []} ] + }, + { + "name": "datadog.trace.bootstrap.otel.shim.metrics.JvmOtlpRuntimeMetrics", + "methods": [ + {"name": "", "parameterTypes": []}, + {"name": "start", "parameterTypes": []} + ] } ] diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java new file mode 100644 index 00000000000..3325783fbae --- /dev/null +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -0,0 +1,291 @@ +package datadog.opentelemetry.shim.metrics; + +import com.sun.management.OperatingSystemMXBean; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadMXBean; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.ToLongFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registers JVM runtime metrics with OTel-native names against the agent's MeterProvider. See + * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/. + */ +public final class JvmOtlpRuntimeMetrics { + + private static final Logger log = LoggerFactory.getLogger(JvmOtlpRuntimeMetrics.class); + private static final String INSTRUMENTATION_SCOPE = "datadog.jvm.runtime"; + private static final AttributeKey MEMORY_TYPE = AttributeKey.stringKey("jvm.memory.type"); + private static final AttributeKey MEMORY_POOL = + AttributeKey.stringKey("jvm.memory.pool.name"); + private static final AttributeKey BUFFER_POOL = + AttributeKey.stringKey("jvm.buffer.pool.name"); + private static final Attributes HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "heap"); + private static final Attributes NON_HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "non_heap"); + + private static final AtomicBoolean started = new AtomicBoolean(false); + + /** Registers all JVM runtime metric instruments on the OTel MeterProvider. */ + public static void start() { + if (!started.compareAndSet(false, true)) { + return; + } + + try { + Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); + registerMemoryMetrics(meter); + registerBufferMetrics(meter); + registerThreadMetrics(meter); + registerClassLoadingMetrics(meter); + registerCpuMetrics(meter); + log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); + } catch (Exception e) { + log.error("Failed to start JVM OTLP runtime metrics", e); + } + } + + // jvm.gc.duration is excluded — spec requires Histogram, JMX only exposes cumulative time. + + /** + * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, + * jvm.memory.used_after_last_gc — all UpDownCounter per spec. + */ + private static void registerMemoryMetrics(Meter meter) { + MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + List pools = ManagementFactory.getMemoryPoolMXBeans(); + + meter + .upDownCounterBuilder("jvm.memory.used") + .setDescription("Measure of memory used.") + .setUnit("By") + .buildWithCallback( + measurement -> { + measurement.record(memoryBean.getHeapMemoryUsage().getUsed(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getUsed(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + measurement.record(pool.getUsage().getUsed(), poolAttributes(pool)); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.committed") + .setDescription("Measure of memory committed.") + .setUnit("By") + .buildWithCallback( + measurement -> { + measurement.record(memoryBean.getHeapMemoryUsage().getCommitted(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getCommitted(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + measurement.record(pool.getUsage().getCommitted(), poolAttributes(pool)); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.limit") + .setDescription("Measure of max obtainable memory.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapMax = memoryBean.getHeapMemoryUsage().getMax(); + if (heapMax > 0) { + measurement.record(heapMax, HEAP_ATTRS); + } + long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); + if (nonHeapMax > 0) { + measurement.record(nonHeapMax, NON_HEAP_ATTRS); + } + for (MemoryPoolMXBean pool : pools) { + long max = pool.getUsage().getMax(); + if (max > 0) { + measurement.record(max, poolAttributes(pool)); + } + } + }); + + meter + .upDownCounterBuilder("jvm.memory.init") + .setDescription("Measure of initial memory requested.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapInit = memoryBean.getHeapMemoryUsage().getInit(); + if (heapInit > 0) { + measurement.record(heapInit, HEAP_ATTRS); + } + long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); + if (nonHeapInit > 0) { + measurement.record(nonHeapInit, NON_HEAP_ATTRS); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.used_after_last_gc") + .setDescription("Measure of memory used after the most recent garbage collection event.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (MemoryPoolMXBean pool : pools) { + MemoryUsage collectionUsage = pool.getCollectionUsage(); + if (collectionUsage != null && collectionUsage.getUsed() >= 0) { + measurement.record(collectionUsage.getUsed(), poolAttributes(pool)); + } + } + }); + } + + /** jvm.buffer.* (UpDownCounter, Development) — direct + mapped pool metrics. */ + private static void registerBufferMetrics(Meter meter) { + List bufferPools = + ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + bufferPoolMetric( + meter, + "jvm.buffer.memory.used", + "Measure of memory used by buffers.", + "By", + bufferPools, + BufferPoolMXBean::getMemoryUsed); + bufferPoolMetric( + meter, + "jvm.buffer.memory.limit", + "Measure of total memory capacity of buffers.", + "By", + bufferPools, + BufferPoolMXBean::getTotalCapacity); + bufferPoolMetric( + meter, + "jvm.buffer.count", + "Number of buffers in the pool.", + "{buffer}", + bufferPools, + BufferPoolMXBean::getCount); + } + + /** jvm.thread.count (UpDownCounter, Stable). */ + private static void registerThreadMetrics(Meter meter) { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + meter + .upDownCounterBuilder("jvm.thread.count") + .setDescription("Number of executing platform threads.") + .setUnit("{thread}") + .buildWithCallback(measurement -> measurement.record(threadBean.getThreadCount())); + } + + /** + * jvm.class.loaded (Counter), jvm.class.unloaded (Counter), jvm.class.count (UpDownCounter) — all + * Stable per spec. + */ + private static void registerClassLoadingMetrics(Meter meter) { + ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean(); + meter + .counterBuilder("jvm.class.loaded") + .setDescription("Number of classes loaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getTotalLoadedClassCount())); + + meter + .upDownCounterBuilder("jvm.class.count") + .setDescription("Number of classes currently loaded.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getLoadedClassCount())); + + meter + .counterBuilder("jvm.class.unloaded") + .setDescription("Number of classes unloaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getUnloadedClassCount())); + } + + /** + * jvm.cpu.time (Counter), jvm.cpu.count (UpDownCounter), jvm.cpu.recent_utilization (Gauge) — all + * Stable per spec. + */ + private static void registerCpuMetrics(Meter meter) { + java.lang.management.OperatingSystemMXBean rawOsBean = + ManagementFactory.getOperatingSystemMXBean(); + OperatingSystemMXBean osBean = + rawOsBean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) rawOsBean : null; + + if (osBean != null) { + meter + .counterBuilder("jvm.cpu.time") + .ofDoubles() + .setDescription("CPU time used by the process as reported by the JVM.") + .setUnit("s") + .buildWithCallback( + measurement -> { + long nanos = osBean.getProcessCpuTime(); + if (nanos >= 0) { + measurement.record(nanos / 1e9); + } + }); + + meter + .gaugeBuilder("jvm.cpu.recent_utilization") + .setDescription("Recent CPU utilization for the process as reported by the JVM.") + .setUnit("1") + .buildWithCallback( + measurement -> { + double cpuLoad = osBean.getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + }); + } + + meter + .upDownCounterBuilder("jvm.cpu.count") + .setDescription("Number of processors available to the JVM.") + .setUnit("{cpu}") + .buildWithCallback( + measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); + } + + /** + * Builds an UpDownCounter that iterates each platform buffer pool and records {@code getter} with + * the {@code jvm.buffer.pool.name} attribute. Skips negative readings. + */ + private static void bufferPoolMetric( + Meter meter, + String name, + String description, + String unit, + List bufferPools, + ToLongFunction getter) { + meter + .upDownCounterBuilder(name) + .setDescription(description) + .setUnit(unit) + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long value = getter.applyAsLong(pool); + if (value >= 0) { + measurement.record(value, Attributes.of(BUFFER_POOL, pool.getName())); + } + } + }); + } + + /** Returns Attributes carrying jvm.memory.type and jvm.memory.pool.name for the given pool. */ + private static Attributes poolAttributes(MemoryPoolMXBean pool) { + return Attributes.of( + MEMORY_TYPE, pool.getType().name().toLowerCase(Locale.ROOT), + MEMORY_POOL, pool.getName()); + } + + private JvmOtlpRuntimeMetrics() {} +} \ No newline at end of file From 5e8567fd33f36d7026e5a3fb28fc506b90bca078 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 19:36:16 -0400 Subject: [PATCH 2/9] Adding test file to check metrics are collected --- .../metrics/JvmOtlpRuntimeMetricsTest.java | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java new file mode 100644 index 00000000000..d0f5bd3795f --- /dev/null +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -0,0 +1,209 @@ +package opentelemetry147.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; +import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry; +import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpDoublePoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Tests that JVM runtime metrics are registered and exported via OTLP using OTel semantic + * convention names (jvm.memory.used, jvm.thread.count, etc.). + * + *

Ref: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + * + *

Ref: + * https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + */ +public class JvmOtlpRuntimeMetricsTest { + + @BeforeAll + static void setUp() { + System.setProperty("dd.metrics.otel.enabled", "true"); + JvmOtlpRuntimeMetrics.start(); + } + + @Test + void registersExpectedJvmMetrics() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List expectedMetrics = + Arrays.asList( + "jvm.memory.used", + "jvm.memory.committed", + "jvm.memory.limit", + "jvm.memory.init", + "jvm.memory.used_after_last_gc", + "jvm.buffer.memory.used", + "jvm.buffer.memory.limit", + "jvm.buffer.count", + "jvm.thread.count", + "jvm.class.loaded", + "jvm.class.count", + "jvm.class.unloaded", + "jvm.cpu.time", + "jvm.cpu.count", + "jvm.cpu.recent_utilization"); + + Set names = collector.metricNames; + for (String metric : expectedMetrics) { + assertTrue( + names.contains(metric), + "Expected metric '" + metric + "' not found. Got: " + new TreeSet<>(names)); + } + + assertEquals(15, names.size(), "Expected 15 metrics, got: " + new TreeSet<>(names)); + + // No DD-proprietary names should be present + List ddNames = + names.stream() + .filter(n -> n.startsWith("jvm.heap_memory") || n.startsWith("jvm.thread_count")) + .collect(Collectors.toList()); + assertTrue(ddNames.isEmpty(), "DD-proprietary names leaked: " + ddNames); + } + + @Test + void jvmMemoryUsedHasHeapAndNonHeapTypeAttributes() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + Set types = collector.attributeValues("jvm.memory.used", "jvm.memory.type"); + assertTrue(types.contains("heap"), "jvm.memory.used should have heap attribute"); + assertTrue(types.contains("non_heap"), "jvm.memory.used should have non_heap attribute"); + } + + @Test + void jvmMemoryUsedHeapValueIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List points = collector.points.get("jvm.memory.used"); + assertNotNull(points, "jvm.memory.used should have data points"); + List heapPoints = + points.stream() + .filter(p -> "heap".equals(p.attrs.get("jvm.memory.type"))) + .collect(Collectors.toList()); + assertFalse(heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); + assertTrue( + heapPoints.get(0).value.longValue() > 0, + "jvm.memory.used heap value should be positive, got " + heapPoints.get(0).value); + } + + @Test + void jvmThreadCountIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List threadPoints = collector.points.get("jvm.thread.count"); + assertNotNull(threadPoints, "jvm.thread.count should have data points"); + assertFalse(threadPoints.isEmpty(), "jvm.thread.count should have data points"); + assertTrue( + threadPoints.get(0).value.longValue() > 0, + "jvm.thread.count value should be positive, got " + threadPoints.get(0).value); + } + + @Test + void startIsIdempotent() { + MetricCollector before = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(before); + int countBefore = before.metricNames.size(); + + JvmOtlpRuntimeMetrics.start(); + JvmOtlpRuntimeMetrics.start(); + + MetricCollector after = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(after); + assertEquals( + countBefore, + after.metricNames.size(), + "Repeated start() must not register duplicate instruments"); + } + + static final class DataPointEntry { + final Map attrs; + final Number value; + + DataPointEntry(Map attrs, Number value) { + this.attrs = attrs; + this.value = value; + } + } + + static final class MetricCollector + implements OtlpMetricsVisitor, OtlpScopedMetricsVisitor, OtlpMetricVisitor { + + String currentInstrument = ""; + final Map currentAttrs = new LinkedHashMap<>(); + final Set metricNames = new LinkedHashSet<>(); + final Map> points = new LinkedHashMap<>(); + + @Override + public OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scope) { + return this; + } + + @Override + public OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { + currentInstrument = descriptor.getName().toString(); + metricNames.add(currentInstrument); + points.computeIfAbsent(currentInstrument, k -> new ArrayList<>()); + return this; + } + + @Override + public void visitAttribute(int type, String key, Object value) { + currentAttrs.put(key, value == null ? null : value.toString()); + } + + @Override + public void visitDataPoint(OtlpDataPoint point) { + Map attrs = new HashMap<>(currentAttrs); + currentAttrs.clear(); + Number value = 0; + if (point instanceof OtlpLongPoint) { + value = ((OtlpLongPoint) point).value; + } else if (point instanceof OtlpDoublePoint) { + value = ((OtlpDoublePoint) point).value; + } + points + .computeIfAbsent(currentInstrument, k -> new ArrayList<>()) + .add(new DataPointEntry(attrs, value)); + } + + Set attributeValues(String metricName, String attrKey) { + List entries = points.get(metricName); + if (entries == null) { + return new LinkedHashSet<>(); + } + return entries.stream() + .map(e -> e.attrs.get(attrKey)) + .filter(Objects::nonNull) + .map(Object::toString) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + } +} \ No newline at end of file From 4a2901ffaf0350e2c0de47d7b5353df71b5f4fa4 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 19:41:24 -0400 Subject: [PATCH 3/9] Doing clean up after testing --- .../opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java | 2 +- .../opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 3325783fbae..6b151ae7370 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -288,4 +288,4 @@ private static Attributes poolAttributes(MemoryPoolMXBean pool) { } private JvmOtlpRuntimeMetrics() {} -} \ No newline at end of file +} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index d0f5bd3795f..72ddfd3c840 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -206,4 +206,4 @@ Set attributeValues(String metricName, String attrKey) { .collect(Collectors.toCollection(LinkedHashSet::new)); } } -} \ No newline at end of file +} From 97592926b47bf3adab598df7f47ce6685372ee1e Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Thu, 14 May 2026 13:06:57 -0400 Subject: [PATCH 4/9] move JvmOtlpRuntimeMetrics.java to agent-jmxfetch --- .../java/datadog/trace/bootstrap/Agent.java | 33 ------------------- .../dd-java-agent/reflect-config.json | 7 ---- dd-java-agent/agent-jmxfetch/build.gradle | 6 ++++ .../trace/agent/jmxfetch/JMXFetch.java | 10 ++++++ .../jmxfetch}/JvmOtlpRuntimeMetrics.java | 3 +- .../opentelemetry-1.47/build.gradle | 2 ++ .../metrics/JvmOtlpRuntimeMetricsTest.java | 2 +- 7 files changed, 21 insertions(+), 42 deletions(-) rename dd-java-agent/{agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics => agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch}/JvmOtlpRuntimeMetrics.java (99%) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 1fa98a07ac6..5288f92dbe3 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -26,7 +26,6 @@ import datadog.instrument.utils.ClassLoaderValue; import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; -import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.Platform; import datadog.trace.api.WithGlobalTracer; import datadog.trace.api.appsec.AppSecEventTracker; @@ -847,17 +846,6 @@ private static synchronized void installDatadogTracer( initTelemetry.onFatalError(ex); } - // Register JVM runtime metric callbacks against the OtelMeterProvider after - // CoreTracer has started OtlpMetricsService. Skip when OTEL_METRICS_EXPORTER=none - // since there's no exporter to collect against. Done here (not in the delayed - // startJmx) so callbacks are in place before the exporter's first flush. - Config cfg = Config.get(); - if (cfg.isRuntimeMetricsEnabled() - && InstrumenterConfig.get().isMetricsOtelEnabled() - && cfg.isMetricsOtlpExporterEnabled()) { - startOtlpRuntimeMetrics(); - } - StaticEventLogger.end("GlobalTracer"); } @@ -1001,27 +989,6 @@ private static synchronized void initializeJmxSystemAccessProvider( } } - /** - * Registers OTLP runtime metric callbacks (JVM heap, CPU, threads, classes, etc.) with the - * agent's OtelMeterProvider. The periodic OTLP exporter started by CoreTracer then collects and - * exports them — this is the same pattern Node and .NET use to start their runtime metrics - * unconditionally during tracer init, independent of any app-side OTel API usage. - */ - private static synchronized void startOtlpRuntimeMetrics() { - final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); - final Class jvmOtlpClass = - AGENT_CLASSLOADER.loadClass("datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics"); - final Method startMethod = jvmOtlpClass.getMethod("start"); - startMethod.invoke(null); - } catch (final Throwable ex) { - log.error("Throwable thrown while starting OTLP runtime metrics", ex); - } finally { - safelySetContextClassLoader(contextLoader); - } - } - private static synchronized void startJmxFetch() { final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); try { diff --git a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json index 2865cfad830..3225d6dd59f 100644 --- a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json +++ b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json @@ -218,12 +218,5 @@ "methods": [ {"name": "", "parameterTypes": []} ] - }, - { - "name": "datadog.trace.bootstrap.otel.shim.metrics.JvmOtlpRuntimeMetrics", - "methods": [ - {"name": "", "parameterTypes": []}, - {"name": "start", "parameterTypes": []} - ] } ] diff --git a/dd-java-agent/agent-jmxfetch/build.gradle b/dd-java-agent/agent-jmxfetch/build.gradle index b65d3a110f5..69dde334306 100644 --- a/dd-java-agent/agent-jmxfetch/build.gradle +++ b/dd-java-agent/agent-jmxfetch/build.gradle @@ -25,6 +25,12 @@ dependencies { api libs.slf4j api project(':internal-api') api project(':dd-java-agent:agent-bootstrap') + + // JvmOtlpRuntimeMetrics registers JVM runtime instruments against the agent's OTel + // MeterProvider. Both deps are compileOnly: the OTel API and OtelMeterProvider are + // bundled into the agent shadow jar via otel-bootstrap / otel-shim from dd-java-agent. + compileOnly group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.47.0' + compileOnly project(':dd-java-agent:agent-otel:otel-shim') } tasks.named("shadowJar", ShadowJar) { diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index f0a2a5541f8..4c9d765097d 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -9,6 +9,7 @@ import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; import datadog.trace.api.GlobalTracer; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.flare.TracerFlare; import datadog.trace.api.telemetry.LogCollector; import de.thetaphi.forbiddenapis.SuppressForbidden; @@ -47,6 +48,15 @@ private static void run(final StatsDClientManager statsDClientManager, final Con return; } + // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP + // exporter started by CoreTracer collects them. Started here so any JMX bootstrap + // side-effects ride the same delayed-start path as JMXFetch itself. + if (config.isRuntimeMetricsEnabled() + && InstrumenterConfig.get().isMetricsOtelEnabled() + && config.isMetricsOtlpExporterEnabled()) { + JvmOtlpRuntimeMetrics.start(); + } + if (!log.isDebugEnabled() && SystemProperties.get("org.slf4j.simpleLogger.log.org.datadog.jmxfetch") == null) { // Reduce noisiness of jmxfetch logging. diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java similarity index 99% rename from dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java rename to dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java index 6b151ae7370..f0af019f086 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java @@ -1,6 +1,7 @@ -package datadog.opentelemetry.shim.metrics; +package datadog.trace.agent.jmxfetch; import com.sun.management.OperatingSystemMXBean; +import datadog.opentelemetry.shim.metrics.OtelMeterProvider; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/build.gradle b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/build.gradle index 3b992b19ff1..ebc3a17f87c 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/build.gradle +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/build.gradle @@ -22,6 +22,8 @@ dependencies { muzzleBootstrap project(path: ':dd-java-agent:agent-otel:otel-bootstrap', configuration: 'shadow') testImplementation project(path: ':dd-java-agent:agent-otel:otel-bootstrap', configuration: 'shadow') + // JvmOtlpRuntimeMetrics lives in agent-jmxfetch; the test exercises it directly. + testImplementation project(':dd-java-agent:agent-jmxfetch') testImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: openTelemetryVersion latestDepTestImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1+' diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index 72ddfd3c840..0d331e9ddf0 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; +import datadog.trace.agent.jmxfetch.JvmOtlpRuntimeMetrics; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry; From 729a87eb8a2f24901ddf9b64549213bf8410965a Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Thu, 14 May 2026 16:43:49 -0400 Subject: [PATCH 5/9] prevent JMXFetch from emitting jvm metrics when otlp is enabled; migrate from depending on otel-shim to otel-bootstrap --- dd-java-agent/agent-jmxfetch/build.gradle | 10 +- .../trace/agent/jmxfetch/JMXFetch.java | 4 +- .../agent/jmxfetch/JvmOtlpRuntimeMetrics.java | 385 ++++++++++-------- .../jmxfetch-config-no-jvm-defaults.yaml | 9 + .../metrics/data/OtelRunnableObservable.java | 15 + .../ResourcesFeatureInstrumentation.java | 1 + 6 files changed, 256 insertions(+), 168 deletions(-) create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/jmxfetch-config-no-jvm-defaults.yaml create mode 100644 dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java diff --git a/dd-java-agent/agent-jmxfetch/build.gradle b/dd-java-agent/agent-jmxfetch/build.gradle index 69dde334306..8eba4b9222e 100644 --- a/dd-java-agent/agent-jmxfetch/build.gradle +++ b/dd-java-agent/agent-jmxfetch/build.gradle @@ -26,11 +26,13 @@ dependencies { api project(':internal-api') api project(':dd-java-agent:agent-bootstrap') - // JvmOtlpRuntimeMetrics registers JVM runtime instruments against the agent's OTel - // MeterProvider. Both deps are compileOnly: the OTel API and OtelMeterProvider are - // bundled into the agent shadow jar via otel-bootstrap / otel-shim from dd-java-agent. + // The opentelemetry-api compileOnly is only for io.opentelemetry.api.common.Attributes / + // AttributeKey used to label the recorded measurements; everything else comes from otel-bootstrap. compileOnly group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.47.0' - compileOnly project(':dd-java-agent:agent-otel:otel-shim') + // JvmOtlpRuntimeMetrics registers JVM runtime instruments directly against the + // bootstrap-level OTel metric registry. otel-bootstrap vendors and repackages the + // OTel API at build time so this won't conflict with anything in the customer app. + compileOnly project(':dd-java-agent:agent-otel:otel-bootstrap') } tasks.named("shadowJar", ShadowJar) { diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 4c9d765097d..8110d2836d8 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -51,9 +51,7 @@ private static void run(final StatsDClientManager statsDClientManager, final Con // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP // exporter started by CoreTracer collects them. Started here so any JMX bootstrap // side-effects ride the same delayed-start path as JMXFetch itself. - if (config.isRuntimeMetricsEnabled() - && InstrumenterConfig.get().isMetricsOtelEnabled() - && config.isMetricsOtlpExporterEnabled()) { + if (InstrumenterConfig.get().isMetricsOtelEnabled() && config.isMetricsOtlpExporterEnabled()) { JvmOtlpRuntimeMetrics.start(); } diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java index f0af019f086..005cb743d1c 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java @@ -1,10 +1,19 @@ package datadog.trace.agent.jmxfetch; +import static datadog.trace.bootstrap.otel.metrics.OtelInstrumentType.COUNTER; +import static datadog.trace.bootstrap.otel.metrics.OtelInstrumentType.GAUGE; +import static datadog.trace.bootstrap.otel.metrics.OtelInstrumentType.UP_DOWN_COUNTER; + import com.sun.management.OperatingSystemMXBean; -import datadog.opentelemetry.shim.metrics.OtelMeterProvider; +import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentBuilder; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentType; +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry; +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricStorage; +import datadog.trace.bootstrap.otel.metrics.data.OtelRunnableObservable; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.Meter; import java.lang.management.BufferPoolMXBean; import java.lang.management.ClassLoadingMXBean; import java.lang.management.ManagementFactory; @@ -15,18 +24,21 @@ import java.util.List; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.ToLongFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Registers JVM runtime metrics with OTel-native names against the agent's MeterProvider. See - * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/. + * Registers JVM runtime metrics with OTel-native names against the agent's bootstrap-level metric + * registry. */ public final class JvmOtlpRuntimeMetrics { private static final Logger log = LoggerFactory.getLogger(JvmOtlpRuntimeMetrics.class); - private static final String INSTRUMENTATION_SCOPE = "datadog.jvm.runtime"; + private static final OtelInstrumentationScope SCOPE = + new OtelInstrumentationScope("datadog.jvm.runtime", null, null); private static final AttributeKey MEMORY_TYPE = AttributeKey.stringKey("jvm.memory.type"); private static final AttributeKey MEMORY_POOL = AttributeKey.stringKey("jvm.memory.pool.name"); @@ -37,19 +49,27 @@ public final class JvmOtlpRuntimeMetrics { private static final AtomicBoolean started = new AtomicBoolean(false); - /** Registers all JVM runtime metric instruments on the OTel MeterProvider. */ + /** Registers all JVM runtime metric instruments on the bootstrap-level metric registry. */ public static void start() { if (!started.compareAndSet(false, true)) { return; } try { - Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); - registerMemoryMetrics(meter); - registerBufferMetrics(meter); - registerThreadMetrics(meter); - registerClassLoadingMetrics(meter); - registerCpuMetrics(meter); + // Ensure OtelMetricStorage can serialize io.opentelemetry.api Attributes recorded below; + // the otel-shim registers an equivalent reader on its own class-loader, but agent-jmxfetch + // does not depend on the shim — so we register one here for our Attributes class-loader. + OtelMetricStorage.registerAttributeReader( + Attributes.class.getClassLoader(), + (attributes, visitor) -> + ((Attributes) attributes) + .forEach((a, v) -> visitor.visitAttribute(a.getType().ordinal(), a.getKey(), v))); + + registerMemoryMetrics(); + registerBufferMetrics(); + registerThreadMetrics(); + registerClassLoadingMetrics(); + registerCpuMetrics(); log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); } catch (Exception e) { log.error("Failed to start JVM OTLP runtime metrics", e); @@ -62,109 +82,106 @@ public static void start() { * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, * jvm.memory.used_after_last_gc — all UpDownCounter per spec. */ - private static void registerMemoryMetrics(Meter meter) { + private static void registerMemoryMetrics() { MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); List pools = ManagementFactory.getMemoryPoolMXBeans(); - meter - .upDownCounterBuilder("jvm.memory.used") - .setDescription("Measure of memory used.") - .setUnit("By") - .buildWithCallback( - measurement -> { - measurement.record(memoryBean.getHeapMemoryUsage().getUsed(), HEAP_ATTRS); - measurement.record(memoryBean.getNonHeapMemoryUsage().getUsed(), NON_HEAP_ATTRS); - for (MemoryPoolMXBean pool : pools) { - measurement.record(pool.getUsage().getUsed(), poolAttributes(pool)); - } - }); + registerLongObservable( + "jvm.memory.used", + "Measure of memory used.", + "By", + UP_DOWN_COUNTER, + storage -> { + storage.recordLong(memoryBean.getHeapMemoryUsage().getUsed(), HEAP_ATTRS); + storage.recordLong(memoryBean.getNonHeapMemoryUsage().getUsed(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + storage.recordLong(pool.getUsage().getUsed(), poolAttributes(pool)); + } + }); - meter - .upDownCounterBuilder("jvm.memory.committed") - .setDescription("Measure of memory committed.") - .setUnit("By") - .buildWithCallback( - measurement -> { - measurement.record(memoryBean.getHeapMemoryUsage().getCommitted(), HEAP_ATTRS); - measurement.record(memoryBean.getNonHeapMemoryUsage().getCommitted(), NON_HEAP_ATTRS); - for (MemoryPoolMXBean pool : pools) { - measurement.record(pool.getUsage().getCommitted(), poolAttributes(pool)); - } - }); + registerLongObservable( + "jvm.memory.committed", + "Measure of memory committed.", + "By", + UP_DOWN_COUNTER, + storage -> { + storage.recordLong(memoryBean.getHeapMemoryUsage().getCommitted(), HEAP_ATTRS); + storage.recordLong(memoryBean.getNonHeapMemoryUsage().getCommitted(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + storage.recordLong(pool.getUsage().getCommitted(), poolAttributes(pool)); + } + }); - meter - .upDownCounterBuilder("jvm.memory.limit") - .setDescription("Measure of max obtainable memory.") - .setUnit("By") - .buildWithCallback( - measurement -> { - long heapMax = memoryBean.getHeapMemoryUsage().getMax(); - if (heapMax > 0) { - measurement.record(heapMax, HEAP_ATTRS); - } - long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); - if (nonHeapMax > 0) { - measurement.record(nonHeapMax, NON_HEAP_ATTRS); - } - for (MemoryPoolMXBean pool : pools) { - long max = pool.getUsage().getMax(); - if (max > 0) { - measurement.record(max, poolAttributes(pool)); - } - } - }); + registerLongObservable( + "jvm.memory.limit", + "Measure of max obtainable memory.", + "By", + UP_DOWN_COUNTER, + storage -> { + long heapMax = memoryBean.getHeapMemoryUsage().getMax(); + if (heapMax > 0) { + storage.recordLong(heapMax, HEAP_ATTRS); + } + long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); + if (nonHeapMax > 0) { + storage.recordLong(nonHeapMax, NON_HEAP_ATTRS); + } + for (MemoryPoolMXBean pool : pools) { + long max = pool.getUsage().getMax(); + if (max > 0) { + storage.recordLong(max, poolAttributes(pool)); + } + } + }); - meter - .upDownCounterBuilder("jvm.memory.init") - .setDescription("Measure of initial memory requested.") - .setUnit("By") - .buildWithCallback( - measurement -> { - long heapInit = memoryBean.getHeapMemoryUsage().getInit(); - if (heapInit > 0) { - measurement.record(heapInit, HEAP_ATTRS); - } - long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); - if (nonHeapInit > 0) { - measurement.record(nonHeapInit, NON_HEAP_ATTRS); - } - }); + registerLongObservable( + "jvm.memory.init", + "Measure of initial memory requested.", + "By", + UP_DOWN_COUNTER, + storage -> { + long heapInit = memoryBean.getHeapMemoryUsage().getInit(); + if (heapInit > 0) { + storage.recordLong(heapInit, HEAP_ATTRS); + } + long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); + if (nonHeapInit > 0) { + storage.recordLong(nonHeapInit, NON_HEAP_ATTRS); + } + }); - meter - .upDownCounterBuilder("jvm.memory.used_after_last_gc") - .setDescription("Measure of memory used after the most recent garbage collection event.") - .setUnit("By") - .buildWithCallback( - measurement -> { - for (MemoryPoolMXBean pool : pools) { - MemoryUsage collectionUsage = pool.getCollectionUsage(); - if (collectionUsage != null && collectionUsage.getUsed() >= 0) { - measurement.record(collectionUsage.getUsed(), poolAttributes(pool)); - } - } - }); + registerLongObservable( + "jvm.memory.used_after_last_gc", + "Measure of memory used after the most recent garbage collection event.", + "By", + UP_DOWN_COUNTER, + storage -> { + for (MemoryPoolMXBean pool : pools) { + MemoryUsage collectionUsage = pool.getCollectionUsage(); + if (collectionUsage != null && collectionUsage.getUsed() >= 0) { + storage.recordLong(collectionUsage.getUsed(), poolAttributes(pool)); + } + } + }); } /** jvm.buffer.* (UpDownCounter, Development) — direct + mapped pool metrics. */ - private static void registerBufferMetrics(Meter meter) { + private static void registerBufferMetrics() { List bufferPools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); bufferPoolMetric( - meter, "jvm.buffer.memory.used", "Measure of memory used by buffers.", "By", bufferPools, BufferPoolMXBean::getMemoryUsed); bufferPoolMetric( - meter, "jvm.buffer.memory.limit", "Measure of total memory capacity of buffers.", "By", bufferPools, BufferPoolMXBean::getTotalCapacity); bufferPoolMetric( - meter, "jvm.buffer.count", "Number of buffers in the pool.", "{buffer}", @@ -173,112 +190,158 @@ private static void registerBufferMetrics(Meter meter) { } /** jvm.thread.count (UpDownCounter, Stable). */ - private static void registerThreadMetrics(Meter meter) { + private static void registerThreadMetrics() { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - meter - .upDownCounterBuilder("jvm.thread.count") - .setDescription("Number of executing platform threads.") - .setUnit("{thread}") - .buildWithCallback(measurement -> measurement.record(threadBean.getThreadCount())); + registerLongObservable( + "jvm.thread.count", + "Number of executing platform threads.", + "{thread}", + UP_DOWN_COUNTER, + storage -> storage.recordLong(threadBean.getThreadCount(), Attributes.empty())); } /** * jvm.class.loaded (Counter), jvm.class.unloaded (Counter), jvm.class.count (UpDownCounter) — all * Stable per spec. */ - private static void registerClassLoadingMetrics(Meter meter) { + private static void registerClassLoadingMetrics() { ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean(); - meter - .counterBuilder("jvm.class.loaded") - .setDescription("Number of classes loaded since JVM start.") - .setUnit("{class}") - .buildWithCallback( - measurement -> measurement.record(classLoadingBean.getTotalLoadedClassCount())); + registerLongObservable( + "jvm.class.loaded", + "Number of classes loaded since JVM start.", + "{class}", + COUNTER, + storage -> + storage.recordLong(classLoadingBean.getTotalLoadedClassCount(), Attributes.empty())); - meter - .upDownCounterBuilder("jvm.class.count") - .setDescription("Number of classes currently loaded.") - .setUnit("{class}") - .buildWithCallback( - measurement -> measurement.record(classLoadingBean.getLoadedClassCount())); + registerLongObservable( + "jvm.class.count", + "Number of classes currently loaded.", + "{class}", + UP_DOWN_COUNTER, + storage -> storage.recordLong(classLoadingBean.getLoadedClassCount(), Attributes.empty())); - meter - .counterBuilder("jvm.class.unloaded") - .setDescription("Number of classes unloaded since JVM start.") - .setUnit("{class}") - .buildWithCallback( - measurement -> measurement.record(classLoadingBean.getUnloadedClassCount())); + registerLongObservable( + "jvm.class.unloaded", + "Number of classes unloaded since JVM start.", + "{class}", + COUNTER, + storage -> + storage.recordLong(classLoadingBean.getUnloadedClassCount(), Attributes.empty())); } /** * jvm.cpu.time (Counter), jvm.cpu.count (UpDownCounter), jvm.cpu.recent_utilization (Gauge) — all * Stable per spec. */ - private static void registerCpuMetrics(Meter meter) { + private static void registerCpuMetrics() { java.lang.management.OperatingSystemMXBean rawOsBean = ManagementFactory.getOperatingSystemMXBean(); OperatingSystemMXBean osBean = rawOsBean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) rawOsBean : null; if (osBean != null) { - meter - .counterBuilder("jvm.cpu.time") - .ofDoubles() - .setDescription("CPU time used by the process as reported by the JVM.") - .setUnit("s") - .buildWithCallback( - measurement -> { - long nanos = osBean.getProcessCpuTime(); - if (nanos >= 0) { - measurement.record(nanos / 1e9); - } - }); + registerDoubleObservable( + "jvm.cpu.time", + "CPU time used by the process as reported by the JVM.", + "s", + COUNTER, + OtelMetricStorage::newDoubleSumStorage, + storage -> { + long nanos = osBean.getProcessCpuTime(); + if (nanos >= 0) { + storage.recordDouble(nanos / 1e9, Attributes.empty()); + } + }); - meter - .gaugeBuilder("jvm.cpu.recent_utilization") - .setDescription("Recent CPU utilization for the process as reported by the JVM.") - .setUnit("1") - .buildWithCallback( - measurement -> { - double cpuLoad = osBean.getProcessCpuLoad(); - if (cpuLoad >= 0) { - measurement.record(cpuLoad); - } - }); + registerDoubleObservable( + "jvm.cpu.recent_utilization", + "Recent CPU utilization for the process as reported by the JVM.", + "1", + GAUGE, + OtelMetricStorage::newDoubleValueStorage, + storage -> { + double cpuLoad = osBean.getProcessCpuLoad(); + if (cpuLoad >= 0) { + storage.recordDouble(cpuLoad, Attributes.empty()); + } + }); } - meter - .upDownCounterBuilder("jvm.cpu.count") - .setDescription("Number of processors available to the JVM.") - .setUnit("{cpu}") - .buildWithCallback( - measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); + registerLongObservable( + "jvm.cpu.count", + "Number of processors available to the JVM.", + "{cpu}", + UP_DOWN_COUNTER, + storage -> + storage.recordLong(Runtime.getRuntime().availableProcessors(), Attributes.empty())); } /** - * Builds an UpDownCounter that iterates each platform buffer pool and records {@code getter} with - * the {@code jvm.buffer.pool.name} attribute. Skips negative readings. + * Registers an UpDownCounter that iterates each platform buffer pool and records {@code getter} + * with the {@code jvm.buffer.pool.name} attribute. Skips negative readings. */ private static void bufferPoolMetric( - Meter meter, String name, String description, String unit, List bufferPools, ToLongFunction getter) { - meter - .upDownCounterBuilder(name) - .setDescription(description) - .setUnit(unit) - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long value = getter.applyAsLong(pool); - if (value >= 0) { - measurement.record(value, Attributes.of(BUFFER_POOL, pool.getName())); - } - } - }); + registerLongObservable( + name, + description, + unit, + UP_DOWN_COUNTER, + storage -> { + for (BufferPoolMXBean pool : bufferPools) { + long value = getter.applyAsLong(pool); + if (value >= 0) { + storage.recordLong(value, Attributes.of(BUFFER_POOL, pool.getName())); + } + } + }); + } + + /** Registers a long observable instrument and its callback against the bootstrap registry. */ + private static void registerLongObservable( + String name, + String description, + String unit, + OtelInstrumentType type, + Consumer callback) { + registerObservable( + OtelInstrumentBuilder.ofLongs(name, type), + description, + unit, + OtelMetricStorage::newLongSumStorage, + callback); + } + + /** Registers a double observable instrument and its callback against the bootstrap registry. */ + private static void registerDoubleObservable( + String name, + String description, + String unit, + OtelInstrumentType type, + Function storageFactory, + Consumer callback) { + registerObservable( + OtelInstrumentBuilder.ofDoubles(name, type), description, unit, storageFactory, callback); + } + + private static void registerObservable( + OtelInstrumentBuilder builder, + String description, + String unit, + Function storageFactory, + Consumer callback) { + builder.setDescription(description); + builder.setUnit(unit); + OtelMetricStorage storage = + OtelMetricRegistry.INSTANCE.registerStorage( + SCOPE, builder.observableDescriptor(), storageFactory); + OtelMetricRegistry.INSTANCE.registerObservable( + SCOPE, new OtelRunnableObservable(() -> callback.accept(storage))); } /** Returns Attributes carrying jvm.memory.type and jvm.memory.pool.name for the given pool. */ diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/jmxfetch-config-no-jvm-defaults.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/jmxfetch-config-no-jvm-defaults.yaml new file mode 100644 index 00000000000..9cb727267e7 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/jmxfetch-config-no-jvm-defaults.yaml @@ -0,0 +1,9 @@ +init_config: + is_jmx: true + new_gc_metrics: true + +instances: + - jvm_direct: true + name: dd-java-agent default + collect_default_jvm_metrics: false + conf: [] # Intentionally left empty for now diff --git a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java new file mode 100644 index 00000000000..75f6e3039fc --- /dev/null +++ b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java @@ -0,0 +1,15 @@ +package datadog.trace.bootstrap.otel.metrics.data; + +/** {@link OtelObservable} backed by a {@link Runnable}, for call sites that prefer a lambda. */ +public final class OtelRunnableObservable extends OtelObservable { + private final Runnable callback; + + public OtelRunnableObservable(Runnable callback) { + this.callback = callback; + } + + @Override + protected void observeMeasurements() { + callback.run(); + } +} diff --git a/dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java b/dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java index f45917f9e8c..bdc78bed041 100644 --- a/dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java +++ b/dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java @@ -54,6 +54,7 @@ public static void onExit() { // tracer's jmxfetch configs tracerResources.add("jmxfetch/jmxfetch-config.yaml"); + tracerResources.add("jmxfetch/jmxfetch-config-no-jvm-defaults.yaml"); tracerResources.add("jmxfetch/jmxfetch-websphere-config.yaml"); // jmxfetch integrations metricconfigs From 5a3a4d1ee4c67dbb800a6a87978e875815e51362 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Thu, 14 May 2026 16:53:47 -0400 Subject: [PATCH 6/9] update JMXFetch to only emit either OTLP or JMX runtime metrics --- .../trace/agent/jmxfetch/JMXFetch.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 8110d2836d8..9bb0651b0a4 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -48,13 +48,6 @@ private static void run(final StatsDClientManager statsDClientManager, final Con return; } - // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP - // exporter started by CoreTracer collects them. Started here so any JMX bootstrap - // side-effects ride the same delayed-start path as JMXFetch itself. - if (InstrumenterConfig.get().isMetricsOtelEnabled() && config.isMetricsOtlpExporterEnabled()) { - JvmOtlpRuntimeMetrics.start(); - } - if (!log.isDebugEnabled() && SystemProperties.get("org.slf4j.simpleLogger.log.org.datadog.jmxfetch") == null) { // Reduce noisiness of jmxfetch logging. @@ -101,9 +94,16 @@ private static void run(final StatsDClientManager statsDClientManager, final Con final StatsDClient statsd = statsDClientManager.statsDClient(host, port, namedPipe, null, null); final AgentStatsdReporter reporter = new AgentStatsdReporter(statsd); + // When the OTLP exporter is collecting JVM runtime metrics, skip the default JMXFetch + // JVM config to avoid double-reporting. + final boolean otlpRuntimeMetricsEnabled = + InstrumenterConfig.get().isMetricsOtelEnabled() && config.isMetricsOtlpExporterEnabled(); + TracerFlare.addReporter(reporter); final List defaultConfigs = new ArrayList<>(); - defaultConfigs.add(DEFAULT_CONFIG); + if (!otlpRuntimeMetricsEnabled) { + defaultConfigs.add(DEFAULT_CONFIG); + } if (config.isJmxFetchIntegrationEnabled(Collections.singletonList("websphere"), false)) { defaultConfigs.add(WEBSPHERE_CONFIG); } @@ -175,6 +175,14 @@ public void run() { } }); thread.setContextClassLoader(JMXFetch.class.getClassLoader()); + + // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP + // exporter started by CoreTracer collects them. Started here so it rides the same + // delayed-start path as JMXFetch itself. + if (otlpRuntimeMetricsEnabled) { + JvmOtlpRuntimeMetrics.start(); + } + thread.start(); } From 414112df39f55ecae25dc2e7c9754ec1b7566e56 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 15 May 2026 00:20:44 -0400 Subject: [PATCH 7/9] send otlp_jmx_config when otlp runtime metrics enabled --- .../datadog/trace/agent/jmxfetch/JMXFetch.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 9bb0651b0a4..1d4ab048a2b 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -33,6 +33,7 @@ public class JMXFetch { private static final Logger log = LoggerFactory.getLogger(JMXFetch.class); private static final String DEFAULT_CONFIG = "jmxfetch-config.yaml"; + private static final String OTLP_JMX_CONFIG = "jmxfetch-config-no-jvm-defaults.yaml"; private static final String WEBSPHERE_CONFIG = "jmxfetch-websphere-config.yaml"; private static final int DELAY_BETWEEN_RUN_ATTEMPTS = 5000; @@ -101,7 +102,13 @@ private static void run(final StatsDClientManager statsDClientManager, final Con TracerFlare.addReporter(reporter); final List defaultConfigs = new ArrayList<>(); - if (!otlpRuntimeMetricsEnabled) { + if (otlpRuntimeMetricsEnabled) { + // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP + // exporter started by CoreTracer collects them. Started here so it rides the same + // delayed-start path as JMXFetch itself. + JvmOtlpRuntimeMetrics.start(); + defaultConfigs.add(OTLP_JMX_CONFIG); + } else { defaultConfigs.add(DEFAULT_CONFIG); } if (config.isJmxFetchIntegrationEnabled(Collections.singletonList("websphere"), false)) { @@ -175,14 +182,6 @@ public void run() { } }); thread.setContextClassLoader(JMXFetch.class.getClassLoader()); - - // Register JVM runtime metric callbacks against the OtelMeterProvider so the OTLP - // exporter started by CoreTracer collects them. Started here so it rides the same - // delayed-start path as JMXFetch itself. - if (otlpRuntimeMetricsEnabled) { - JvmOtlpRuntimeMetrics.start(); - } - thread.start(); } From 45333511000c383e591cc39f22a9036be9fb9ca6 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 15 May 2026 11:43:54 -0400 Subject: [PATCH 8/9] update test to assert on guarantees instead of dependent on GC collection --- .../metrics/JvmOtlpRuntimeMetricsTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index 0d331e9ddf0..6d220161bee 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -103,14 +103,18 @@ void jvmMemoryUsedHeapValueIsPositive() { List points = collector.points.get("jvm.memory.used"); assertNotNull(points, "jvm.memory.used should have data points"); - List heapPoints = + DataPointEntry heapAggregate = points.stream() - .filter(p -> "heap".equals(p.attrs.get("jvm.memory.type"))) - .collect(Collectors.toList()); - assertFalse(heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); + .filter( + p -> + "heap".equals(p.attrs.get("jvm.memory.type")) + && p.attrs.get("jvm.memory.pool.name") == null) + .findFirst() + .orElse(null); + assertNotNull(heapAggregate, "jvm.memory.used should have a heap aggregate data point"); assertTrue( - heapPoints.get(0).value.longValue() > 0, - "jvm.memory.used heap value should be positive, got " + heapPoints.get(0).value); + heapAggregate.value.longValue() > 0, + "jvm.memory.used heap aggregate should be positive, got " + heapAggregate.value); } @Test From 25a6396119668c7ae470a897855c42d50f3ce20f Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 15 May 2026 15:53:26 -0400 Subject: [PATCH 9/9] adding exception handling for callback --- .../datadog/trace/agent/jmxfetch/JMXFetch.java | 4 ++-- .../agent/jmxfetch/JvmOtlpRuntimeMetrics.java | 2 -- .../otel/metrics/data/OtelRunnableObservable.java | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 1d4ab048a2b..5375fb8b3e6 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -95,8 +95,6 @@ private static void run(final StatsDClientManager statsDClientManager, final Con final StatsDClient statsd = statsDClientManager.statsDClient(host, port, namedPipe, null, null); final AgentStatsdReporter reporter = new AgentStatsdReporter(statsd); - // When the OTLP exporter is collecting JVM runtime metrics, skip the default JMXFetch - // JVM config to avoid double-reporting. final boolean otlpRuntimeMetricsEnabled = InstrumenterConfig.get().isMetricsOtelEnabled() && config.isMetricsOtlpExporterEnabled(); @@ -107,6 +105,8 @@ private static void run(final StatsDClientManager statsDClientManager, final Con // exporter started by CoreTracer collects them. Started here so it rides the same // delayed-start path as JMXFetch itself. JvmOtlpRuntimeMetrics.start(); + // When the OTLP exporter is collecting JVM runtime metrics, skip the default JMXFetch + // JVM config to avoid double-reporting. defaultConfigs.add(OTLP_JMX_CONFIG); } else { defaultConfigs.add(DEFAULT_CONFIG); diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java index 005cb743d1c..696b2345f93 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JvmOtlpRuntimeMetrics.java @@ -76,8 +76,6 @@ public static void start() { } } - // jvm.gc.duration is excluded — spec requires Histogram, JMX only exposes cumulative time. - /** * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, * jvm.memory.used_after_last_gc — all UpDownCounter per spec. diff --git a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java index 75f6e3039fc..975d59f8826 100644 --- a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java +++ b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelRunnableObservable.java @@ -1,7 +1,16 @@ package datadog.trace.bootstrap.otel.metrics.data; +import datadog.logging.RatelimitedLogger; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** {@link OtelObservable} backed by a {@link Runnable}, for call sites that prefer a lambda. */ public final class OtelRunnableObservable extends OtelObservable { + private static final Logger LOGGER = LoggerFactory.getLogger(OtelRunnableObservable.class); + private static final RatelimitedLogger RATELIMITED_LOGGER = + new RatelimitedLogger(LOGGER, 5, TimeUnit.MINUTES); + private final Runnable callback; public OtelRunnableObservable(Runnable callback) { @@ -10,6 +19,10 @@ public OtelRunnableObservable(Runnable callback) { @Override protected void observeMeasurements() { - callback.run(); + try { + callback.run(); + } catch (Throwable e) { + RATELIMITED_LOGGER.warn("An exception occurred invoking observable callback.", e); + } } }