From bc420936c96fcade512ae0e8e75190705371bbd8 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 12:06:49 -0400 Subject: [PATCH 1/5] Trim per-span work on metrics aggregator publish path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConflatingMetricsAggregator.publish does a handful of redundant operations on every span. None individually is large; together they show as ~2.5% on the existing JMH benchmark once the benchmark actually exercises span.kind. - dedup span.isTopLevel(): publish() reads it into a local, then shouldComputeMetric read it again. Pass the cached value in. - resolve spanKind to String once: master called toString() twice per span (once inside spanKindEligible, once at the getPeerTags call site) and used HashSet contains on a CharSequence (which routes through equals on String). Normalize to String up front and reuse. - lazy-allocate the peer-tag list: getPeerTags() always allocated an ArrayList sized to features.peerTags() even when the span had none of those tags set. Defer allocation until the first match; return Collections.emptyList() when none hit. MetricKey already treats null/empty peerTags as emptyList, so no behavior change. Drop the spanKindEligible helper — the HashSet.contains call inlines fine in shouldComputeMetric. Update the JMH benchmark to set span.kind=client on every span. Without it the filter path short-circuits before the peer-tag and toString work, so the wins above aren't measurable. With it: baseline 6.755 us/op (CI [6.560, 6.950], stdev 0.129) optimized 6.585 us/op (CI [6.536, 6.634], stdev 0.033) 2 forks x 5 iterations x 15s. ~2.5% mean improvement and much tighter variance fork-to-fork. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ConflatingMetricsAggregatorBenchmark.java | 3 +++ .../metrics/ConflatingMetricsAggregator.java | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBenchmark.java index 971ee5cf6e4..b9a2f7f8c54 100644 --- a/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBenchmark.java +++ b/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBenchmark.java @@ -1,6 +1,8 @@ package datadog.trace.common.metrics; import static datadog.trace.api.ProtocolVersion.V0_4; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT; import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -52,6 +54,7 @@ static List> generateTrace(int len) { final List> trace = new ArrayList<>(); for (int i = 0; i < len; i++) { SimpleSpan span = new SimpleSpan("", "", "", "", true, true, false, 0, 10, -1); + span.setTag(SPAN_KIND, SPAN_KIND_CLIENT); span.setTag("peer.hostname", Strings.random(10)); trace.add(span); } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index f60edf1d700..408b7688458 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -289,8 +289,10 @@ public boolean publish(List> trace) { if (features.supportsMetrics()) { for (CoreSpan span : trace) { boolean isTopLevel = span.isTopLevel(); - final CharSequence spanKind = span.unsafeGetTag(SPAN_KIND, ""); - if (shouldComputeMetric(span, spanKind)) { + // CharSequence cast keeps unsafeGetTag's generic at CharSequence so UTF8BytesString + // tag values don't trigger a ClassCastException on the String assignment. + final String spanKind = span.unsafeGetTag(SPAN_KIND, (CharSequence) "").toString(); + if (shouldComputeMetric(span, isTopLevel, spanKind)) { final CharSequence resourceName = span.getResourceName(); if (resourceName != null && ignoredResources.contains(resourceName.toString())) { // skip publishing all children @@ -306,19 +308,15 @@ public boolean publish(List> trace) { return forceKeep; } - private boolean shouldComputeMetric(CoreSpan span, @Nonnull CharSequence spanKind) { - return (span.isMeasured() || span.isTopLevel() || spanKindEligible(spanKind)) + private boolean shouldComputeMetric( + CoreSpan span, boolean isTopLevel, @Nonnull String spanKind) { + return (span.isMeasured() || isTopLevel || ELIGIBLE_SPAN_KINDS_FOR_METRICS.contains(spanKind)) && span.getLongRunningVersion() <= 0 // either not long-running or unpublished long-running span && span.getDurationNano() > 0; } - private boolean spanKindEligible(@Nonnull CharSequence spanKind) { - // use toString since it could be a CharSequence... - return ELIGIBLE_SPAN_KINDS_FOR_METRICS.contains(spanKind.toString()); - } - - private boolean publish(CoreSpan span, boolean isTopLevel, CharSequence spanKind) { + private boolean publish(CoreSpan span, boolean isTopLevel, String spanKind) { // Extract HTTP method and endpoint only if the feature is enabled String httpMethod = null; String httpEndpoint = null; @@ -347,7 +345,7 @@ private boolean publish(CoreSpan span, boolean isTopLevel, CharSequence spanK span.getParentId() == 0, SPAN_KINDS.computeIfAbsent( spanKind, UTF8BytesString::create), // save repeated utf8 conversions - getPeerTags(span, spanKind.toString()), + getPeerTags(span, spanKind), httpMethod, httpEndpoint, grpcStatusCode); @@ -385,19 +383,22 @@ private boolean publish(CoreSpan span, boolean isTopLevel, CharSequence spanK private List getPeerTags(CoreSpan span, String spanKind) { if (ELIGIBLE_SPAN_KINDS_FOR_PEER_AGGREGATION.contains(spanKind)) { final Set eligiblePeerTags = features.peerTags(); - List peerTags = new ArrayList<>(eligiblePeerTags.size()); + List peerTags = null; for (String peerTag : eligiblePeerTags) { Object value = span.unsafeGetTag(peerTag); if (value != null) { final Pair, Function> cacheAndCreator = PEER_TAGS_CACHE.computeIfAbsent(peerTag, PEER_TAGS_CACHE_ADDER); + if (peerTags == null) { + peerTags = new ArrayList<>(eligiblePeerTags.size()); + } peerTags.add( cacheAndCreator .getLeft() .computeIfAbsent(value.toString(), cacheAndCreator.getRight())); } } - return peerTags; + return peerTags == null ? Collections.emptyList() : peerTags; } else if (SPAN_KIND_INTERNAL.equals(spanKind)) { // in this case only the base service should be aggregated if present final Object baseService = span.unsafeGetTag(BASE_SERVICE); From 808d63d04c45acc4893d7ac5671b48ab88c6cf86 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 12:34:10 -0400 Subject: [PATCH 2/5] Add SpanKindFilter and CoreSpan.isKind for bitmask-based kind checks Introduce SpanKindFilter -- a tiny builder-built immutable filter whose state is an int bitmask indexed by the span.kind ordinals already cached on DDSpanContext. Each include* on the builder sets one bit (1 << ordinal); the runtime check is a single AND against (1 << span's ordinal). CoreSpan.isKind(SpanKindFilter) is the new entry point. DDSpan overrides it to do the bit-test directly against the cached ordinal -- no virtual call, no tag-map lookup. The two existing test-only CoreSpan impls (SimpleSpan and TraceGenerator.PojoSpan, the latter in two source sets) implement isKind by reading the span.kind tag and delegating to SpanKindFilter.matches(String), which converts via DDSpanContext.spanKindOrdinalOf and does the same AND. Refactor: DDSpanContext.setSpanKindOrdinal(String) now delegates to a new package-private static spanKindOrdinalOf(String) so the same string-to-ordinal mapping serves both the tag interceptor path and SpanKindFilter.matches. This is groundwork -- nothing in the codebase calls isKind yet. The next commit will replace the HashSet-based eligibility checks in ConflatingMetricsAggregator with SpanKindFilter instances. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/core/CoreSpan.java | 2 + .../main/java/datadog/trace/core/DDSpan.java | 4 ++ .../datadog/trace/core/DDSpanContext.java | 20 ++++--- .../datadog/trace/core/SpanKindFilter.java | 55 +++++++++++++++++++ .../trace/common/metrics/SimpleSpan.groovy | 8 +++ .../trace/common/writer/TraceGenerator.groovy | 8 +++ .../groovy/TraceGenerator.groovy | 8 +++ 7 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreSpan.java index 8c98cbbc58a..7d183670883 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreSpan.java @@ -80,6 +80,8 @@ default U unsafeGetTag(CharSequence name) { boolean isForceKeep(); + boolean isKind(SpanKindFilter filter); + CharSequence getType(); /** diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 2c62819e97a..ab074d8d4c8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -959,6 +959,10 @@ public boolean isOutbound() { return ordinal == DDSpanContext.SPAN_KIND_CLIENT || ordinal == DDSpanContext.SPAN_KIND_PRODUCER; } + public boolean isKind(SpanKindFilter filter) { + return (filter.kindMask & (1 << context.getSpanKindOrdinal())) != 0; + } + @Override public void copyPropagationAndBaggage(final AgentSpan source) { if (source instanceof DDSpan) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index f2eb17fe8a2..a7c0849943e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -771,22 +771,26 @@ static boolean tagEquals(String tagValue, String tagLiteral) { * span.kind is set. */ public void setSpanKindOrdinal(String kind) { + spanKindOrdinal = spanKindOrdinalOf(kind); + } + + static byte spanKindOrdinalOf(String kind) { if (kind == null) { - spanKindOrdinal = SPAN_KIND_UNSET; + return SPAN_KIND_UNSET; } else if (tagEquals(kind, Tags.SPAN_KIND_SERVER)) { - spanKindOrdinal = SPAN_KIND_SERVER; + return SPAN_KIND_SERVER; } else if (tagEquals(kind, Tags.SPAN_KIND_CLIENT)) { - spanKindOrdinal = SPAN_KIND_CLIENT; + return SPAN_KIND_CLIENT; } else if (tagEquals(kind, Tags.SPAN_KIND_PRODUCER)) { - spanKindOrdinal = SPAN_KIND_PRODUCER; + return SPAN_KIND_PRODUCER; } else if (tagEquals(kind, Tags.SPAN_KIND_CONSUMER)) { - spanKindOrdinal = SPAN_KIND_CONSUMER; + return SPAN_KIND_CONSUMER; } else if (tagEquals(kind, Tags.SPAN_KIND_INTERNAL)) { - spanKindOrdinal = SPAN_KIND_INTERNAL; + return SPAN_KIND_INTERNAL; } else if (tagEquals(kind, Tags.SPAN_KIND_BROKER)) { - spanKindOrdinal = SPAN_KIND_BROKER; + return SPAN_KIND_BROKER; } else { - spanKindOrdinal = SPAN_KIND_CUSTOM; + return SPAN_KIND_CUSTOM; } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java new file mode 100644 index 00000000000..39ca3031039 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java @@ -0,0 +1,55 @@ +package datadog.trace.core; + +public final class SpanKindFilter { + public static final class Builder { + private int kindMask; + + public Builder includeServer() { + return this.include(DDSpanContext.SPAN_KIND_SERVER); + } + + public Builder includeClient() { + return this.include(DDSpanContext.SPAN_KIND_CLIENT); + } + + public Builder includeProducer() { + return this.include(DDSpanContext.SPAN_KIND_PRODUCER); + } + + public Builder includeConsumer() { + return this.include(DDSpanContext.SPAN_KIND_CONSUMER); + } + + public Builder includeInternal() { + return this.include(DDSpanContext.SPAN_KIND_INTERNAL); + } + + public Builder includeBroker() { + return this.include(DDSpanContext.SPAN_KIND_BROKER); + } + + public final SpanKindFilter build() { + return new SpanKindFilter(this.kindMask); + } + + private Builder include(int spanKindConstant) { + this.kindMask |= (1 << spanKindConstant); + return this; + } + } + + public static final Builder builder() { + return new Builder(); + } + + final int kindMask; + + SpanKindFilter(int kindMask) { + this.kindMask = kindMask; + } + + /** Test whether a span with the given span.kind string passes this filter. */ + public boolean matches(String spanKind) { + return (kindMask & (1 << DDSpanContext.spanKindOrdinalOf(spanKind))) != 0; + } +} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy index bfc1ee2f4e7..61c8597129c 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy @@ -2,8 +2,10 @@ package datadog.trace.common.metrics import datadog.trace.api.DDSpanId import datadog.trace.api.DDTraceId +import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.CoreSpan import datadog.trace.core.MetadataConsumer +import datadog.trace.core.SpanKindFilter class SimpleSpan implements CoreSpan { @@ -211,6 +213,12 @@ class SimpleSpan implements CoreSpan { return false } + @Override + boolean isKind(SpanKindFilter filter) { + def kind = tags.get(Tags.SPAN_KIND) + return filter.matches(kind == null ? null : kind.toString()) + } + @Override CharSequence getType() { return type diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceGenerator.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceGenerator.groovy index 66bdbab137b..49e13472249 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceGenerator.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceGenerator.groovy @@ -11,10 +11,12 @@ import datadog.trace.api.ProcessTags import datadog.trace.api.TagMap import datadog.trace.api.sampling.PrioritySampling import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink +import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString import datadog.trace.core.CoreSpan import datadog.trace.core.Metadata import datadog.trace.core.MetadataConsumer +import datadog.trace.core.SpanKindFilter import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.TimeUnit @@ -321,6 +323,12 @@ class TraceGenerator { return false } + @Override + boolean isKind(SpanKindFilter filter) { + def kind = metadata.getTags().get(Tags.SPAN_KIND) + return filter.matches(kind == null ? null : kind.toString()) + } + @Override short getHttpStatusCode() { return httpStatusCode diff --git a/dd-trace-core/src/traceAgentTest/groovy/TraceGenerator.groovy b/dd-trace-core/src/traceAgentTest/groovy/TraceGenerator.groovy index e668d0112a6..2b2bca79406 100644 --- a/dd-trace-core/src/traceAgentTest/groovy/TraceGenerator.groovy +++ b/dd-trace-core/src/traceAgentTest/groovy/TraceGenerator.groovy @@ -9,10 +9,12 @@ import datadog.trace.api.DDTags import datadog.trace.api.DDTraceId import datadog.trace.api.IdGenerationStrategy import datadog.trace.api.TagMap +import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString import datadog.trace.core.CoreSpan import datadog.trace.core.Metadata import datadog.trace.core.MetadataConsumer +import datadog.trace.core.SpanKindFilter import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.TimeUnit @@ -298,6 +300,12 @@ class TraceGenerator { return false } + @Override + boolean isKind(SpanKindFilter filter) { + def kind = metadata.getTags().get(Tags.SPAN_KIND) + return filter.matches(kind == null ? null : kind.toString()) + } + Map getBaggage() { return metadata.getBaggage() } From 6aa620ec53ce86c5c255a5b437e3c04df251ea83 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 12:58:41 -0400 Subject: [PATCH 3/5] Use SpanKindFilter in ConflatingMetricsAggregator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the two ELIGIBLE_SPAN_KINDS_FOR_* HashSet constants and the SPAN_KIND_INTERNAL.equals check with three SpanKindFilter instances: METRICS_ELIGIBLE_KINDS, PEER_AGGREGATION_KINDS, INTERNAL_KIND. Eligibility checks now go through span.isKind(filter), which on DDSpan is a volatile byte read against the already-cached span.kind ordinal plus a single bit-test. Also defer the span.kind tag read: previously read at the top of the publish loop and threaded through both shouldComputeMetric and the inner publish. isKind no longer needs the string, so the read can move down into the inner publish where it's still needed for the SPAN_KINDS cache key / MetricKey. Supporting changes: - DDSpanContext.spanKindOrdinalOf(String) is now public so non-DDSpan CoreSpan impls can compute the ordinal at tag-write time. - SpanKindFilter gains a public matches(byte) fast-path overload that callers with a pre-computed ordinal use directly. - SimpleSpan caches the ordinal in setTag(SPAN_KIND, ...), mirroring what TagInterceptor does for DDSpanContext, and its isKind now hits the byte fast path. Without this, the JMH benchmark (which uses SimpleSpan) would re-derive the ordinal on every isKind call and overstate the cost. Benchmark on the bench updated last commit (kind=client on every span, 4 forks x 5 iter x 15s): prior commit 6.585 ± 0.049 us/op this commit 6.903 ± 0.096 us/op The slight regression is a SimpleSpan-via-groovy-dispatch artifact -- the interface call to isKind through CoreSpan, then through SimpleSpan, then through SpanKindFilter.matches, doesn't fold as aggressively as a HashSet contains on a static field. In production DDSpan.isKind inlines to a context field read + ordinal byte read + bit-test, so the production path is faster than the prior HashSet approach. A DDSpan-based benchmark would show this; the existing SimpleSpan-based one doesn't. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../metrics/ConflatingMetricsAggregator.java | 55 +++++++++---------- .../datadog/trace/core/DDSpanContext.java | 2 +- .../datadog/trace/core/SpanKindFilter.java | 7 ++- .../trace/common/metrics/SimpleSpan.groovy | 9 ++- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index 408b7688458..fee2f9a7748 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -7,11 +7,6 @@ import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ENDPOINT; import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_METHOD; import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CONSUMER; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_INTERNAL; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_PRODUCER; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_SERVER; import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG; import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG; import static datadog.trace.common.metrics.SignalItem.ReportSignal.REPORT; @@ -19,7 +14,6 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.METRICS_AGGREGATOR; import static datadog.trace.util.AgentThreadFactory.THREAD_JOIN_TIMOUT_MS; import static datadog.trace.util.AgentThreadFactory.newAgentThread; -import static java.util.Collections.unmodifiableSet; import static java.util.concurrent.TimeUnit.SECONDS; import datadog.common.queue.Queues; @@ -36,12 +30,11 @@ import datadog.trace.common.writer.ddagent.DDAgentApi; import datadog.trace.core.CoreSpan; import datadog.trace.core.DDTraceCoreInfo; +import datadog.trace.core.SpanKindFilter; import datadog.trace.core.monitor.HealthMetrics; import datadog.trace.util.AgentTaskScheduler; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,7 +43,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import javax.annotation.Nonnull; import org.jctools.queues.MessagePassingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,15 +74,19 @@ public final class ConflatingMetricsAggregator implements MetricsAggregator, Eve value -> UTF8BytesString.create(key + ":" + value)); private static final CharSequence SYNTHETICS_ORIGIN = "synthetics"; - private static final Set ELIGIBLE_SPAN_KINDS_FOR_METRICS = - unmodifiableSet( - new HashSet<>( - Arrays.asList( - SPAN_KIND_SERVER, SPAN_KIND_CLIENT, SPAN_KIND_CONSUMER, SPAN_KIND_PRODUCER))); + private static final SpanKindFilter METRICS_ELIGIBLE_KINDS = + SpanKindFilter.builder() + .includeServer() + .includeClient() + .includeProducer() + .includeConsumer() + .build(); - private static final Set ELIGIBLE_SPAN_KINDS_FOR_PEER_AGGREGATION = - unmodifiableSet( - new HashSet<>(Arrays.asList(SPAN_KIND_CLIENT, SPAN_KIND_PRODUCER, SPAN_KIND_CONSUMER))); + private static final SpanKindFilter PEER_AGGREGATION_KINDS = + SpanKindFilter.builder().includeClient().includeProducer().includeConsumer().build(); + + private static final SpanKindFilter INTERNAL_KIND = + SpanKindFilter.builder().includeInternal().build(); private final Set ignoredResources; private final MessagePassingQueue batchPool; @@ -289,10 +285,7 @@ public boolean publish(List> trace) { if (features.supportsMetrics()) { for (CoreSpan span : trace) { boolean isTopLevel = span.isTopLevel(); - // CharSequence cast keeps unsafeGetTag's generic at CharSequence so UTF8BytesString - // tag values don't trigger a ClassCastException on the String assignment. - final String spanKind = span.unsafeGetTag(SPAN_KIND, (CharSequence) "").toString(); - if (shouldComputeMetric(span, isTopLevel, spanKind)) { + if (shouldComputeMetric(span, isTopLevel)) { final CharSequence resourceName = span.getResourceName(); if (resourceName != null && ignoredResources.contains(resourceName.toString())) { // skip publishing all children @@ -300,7 +293,7 @@ public boolean publish(List> trace) { break; } counted++; - forceKeep |= publish(span, isTopLevel, spanKind); + forceKeep |= publish(span, isTopLevel); } } healthMetrics.onClientStatTraceComputed(counted, trace.size(), !forceKeep); @@ -308,15 +301,14 @@ public boolean publish(List> trace) { return forceKeep; } - private boolean shouldComputeMetric( - CoreSpan span, boolean isTopLevel, @Nonnull String spanKind) { - return (span.isMeasured() || isTopLevel || ELIGIBLE_SPAN_KINDS_FOR_METRICS.contains(spanKind)) + private boolean shouldComputeMetric(CoreSpan span, boolean isTopLevel) { + return (span.isMeasured() || isTopLevel || span.isKind(METRICS_ELIGIBLE_KINDS)) && span.getLongRunningVersion() <= 0 // either not long-running or unpublished long-running span && span.getDurationNano() > 0; } - private boolean publish(CoreSpan span, boolean isTopLevel, String spanKind) { + private boolean publish(CoreSpan span, boolean isTopLevel) { // Extract HTTP method and endpoint only if the feature is enabled String httpMethod = null; String httpEndpoint = null; @@ -333,6 +325,9 @@ private boolean publish(CoreSpan span, boolean isTopLevel, String spanKind) { Object grpcStatusObj = span.unsafeGetTag(InstrumentationTags.GRPC_STATUS_CODE); grpcStatusCode = grpcStatusObj != null ? grpcStatusObj.toString() : null; } + // CharSequence default keeps unsafeGetTag's generic at CharSequence so UTF8BytesString + // tag values don't trigger a ClassCastException on the String assignment. + final String spanKind = span.unsafeGetTag(SPAN_KIND, (CharSequence) "").toString(); MetricKey newKey = new MetricKey( span.getResourceName(), @@ -345,7 +340,7 @@ private boolean publish(CoreSpan span, boolean isTopLevel, String spanKind) { span.getParentId() == 0, SPAN_KINDS.computeIfAbsent( spanKind, UTF8BytesString::create), // save repeated utf8 conversions - getPeerTags(span, spanKind), + getPeerTags(span), httpMethod, httpEndpoint, grpcStatusCode); @@ -380,8 +375,8 @@ private boolean publish(CoreSpan span, boolean isTopLevel, String spanKind) { return span.getError() > 0; } - private List getPeerTags(CoreSpan span, String spanKind) { - if (ELIGIBLE_SPAN_KINDS_FOR_PEER_AGGREGATION.contains(spanKind)) { + private List getPeerTags(CoreSpan span) { + if (span.isKind(PEER_AGGREGATION_KINDS)) { final Set eligiblePeerTags = features.peerTags(); List peerTags = null; for (String peerTag : eligiblePeerTags) { @@ -399,7 +394,7 @@ private List getPeerTags(CoreSpan span, String spanKind) { } } return peerTags == null ? Collections.emptyList() : peerTags; - } else if (SPAN_KIND_INTERNAL.equals(spanKind)) { + } else if (span.isKind(INTERNAL_KIND)) { // in this case only the base service should be aggregated if present final Object baseService = span.unsafeGetTag(BASE_SERVICE); if (baseService != null) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index a7c0849943e..e403efd543b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -774,7 +774,7 @@ public void setSpanKindOrdinal(String kind) { spanKindOrdinal = spanKindOrdinalOf(kind); } - static byte spanKindOrdinalOf(String kind) { + public static byte spanKindOrdinalOf(String kind) { if (kind == null) { return SPAN_KIND_UNSET; } else if (tagEquals(kind, Tags.SPAN_KIND_SERVER)) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java index 39ca3031039..600e0d9ca47 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java @@ -50,6 +50,11 @@ public static final Builder builder() { /** Test whether a span with the given span.kind string passes this filter. */ public boolean matches(String spanKind) { - return (kindMask & (1 << DDSpanContext.spanKindOrdinalOf(spanKind))) != 0; + return matches(DDSpanContext.spanKindOrdinalOf(spanKind)); + } + + /** Fast-path test for callers that already hold the span's cached kind ordinal. */ + public boolean matches(byte spanKindOrdinal) { + return (kindMask & (1 << spanKindOrdinal)) != 0; } } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy index 61c8597129c..2fd8554d499 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SimpleSpan.groovy @@ -4,6 +4,7 @@ import datadog.trace.api.DDSpanId import datadog.trace.api.DDTraceId import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.CoreSpan +import datadog.trace.core.DDSpanContext import datadog.trace.core.MetadataConsumer import datadog.trace.core.SpanKindFilter @@ -26,6 +27,8 @@ class SimpleSpan implements CoreSpan { private final Map tags = [:] + private byte spanKindOrdinal = 0 // SPAN_KIND_UNSET + SimpleSpan( String serviceName, String operationName, @@ -173,6 +176,9 @@ class SimpleSpan implements CoreSpan { @Override SimpleSpan setTag(String tag, Object value) { tags.put(tag, value) + if (Tags.SPAN_KIND == tag) { + spanKindOrdinal = DDSpanContext.spanKindOrdinalOf(value == null ? null : value.toString()) + } return this } @@ -215,8 +221,7 @@ class SimpleSpan implements CoreSpan { @Override boolean isKind(SpanKindFilter filter) { - def kind = tags.get(Tags.SPAN_KIND) - return filter.matches(kind == null ? null : kind.toString()) + return filter.matches(spanKindOrdinal) } @Override From a02d0a9cb1b8d67d807160cd4c7a796c627143a2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 13:13:07 -0400 Subject: [PATCH 4/5] Add DDSpan-based variant of ConflatingMetricsAggregator JMH benchmark The existing ConflatingMetricsAggregatorBenchmark uses SimpleSpan, a groovy mock. That's enough for measuring queue/CHM/MetricKey work, but it conceals the production cost of CoreSpan.isKind: SimpleSpan's isKind goes through groovy interface dispatch into SpanKindFilter.matches, while DDSpan.isKind inlines to a context byte-read + bit-test. This new benchmark uses real DDSpan instances created through a CoreTracer (with a NoopWriter so finishing doesn't reach the agent). Same shape as the SimpleSpan bench (64-span trace, span.kind=client, peer.hostname set). Numbers (2 forks x 5 iter x 15s): master: 6.428 +- 0.189 us/op (HashSet eligibility checks) this branch: 6.343 +- 0.115 us/op (SpanKindFilter bitmask) About 1.3% faster on the production path. The SimpleSpan benchmark in the same conditions shows a ~2.2% slowdown -- the mock's dispatch shape gives a misleading signal. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...atingMetricsAggregatorDDSpanBenchmark.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDDSpanBenchmark.java diff --git a/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDDSpanBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDDSpanBenchmark.java new file mode 100644 index 00000000000..02c6aaffc1a --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDDSpanBenchmark.java @@ -0,0 +1,98 @@ +package datadog.trace.common.metrics; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.trace.api.WellKnownTags; +import datadog.trace.common.writer.Writer; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.util.Strings; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Parallels {@link ConflatingMetricsAggregatorBenchmark} but uses real {@link DDSpan} instances + * instead of the lightweight {@code SimpleSpan} mock, so the JIT exercises the production {@link + * CoreSpan#isKind} path (cached span.kind ordinal + bit-test) rather than the groovy mock's + * dispatch. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 1, time = 30, timeUnit = SECONDS) +@Measurement(iterations = 3, time = 30, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(MICROSECONDS) +@Fork(value = 1) +public class ConflatingMetricsAggregatorDDSpanBenchmark { + + private static final CoreTracer TRACER = + CoreTracer.builder().writer(new NoopWriter()).strictTraceWrites(false).build(); + + private final DDAgentFeaturesDiscovery featuresDiscovery = + new ConflatingMetricsAggregatorBenchmark.FixedAgentFeaturesDiscovery( + Collections.singleton("peer.hostname"), Collections.emptySet()); + private final ConflatingMetricsAggregator aggregator = + new ConflatingMetricsAggregator( + new WellKnownTags("", "", "", "", "", ""), + Collections.emptySet(), + featuresDiscovery, + HealthMetrics.NO_OP, + new ConflatingMetricsAggregatorBenchmark.NullSink(), + 2048, + 2048, + false); + private final List> spans = generateTrace(64); + + static List> generateTrace(int len) { + final List> trace = new ArrayList<>(); + for (int i = 0; i < len; i++) { + DDSpan span = (DDSpan) TRACER.startSpan("benchmark", "op"); + span.setTag(SPAN_KIND, SPAN_KIND_CLIENT); + span.setTag("peer.hostname", Strings.random(10)); + // Fix duration; bypasses the wall clock and avoids per-fork drift. + span.finishWithDuration(10); + trace.add(span); + } + return trace; + } + + static class NoopWriter implements Writer { + @Override + public void write(List trace) {} + + @Override + public void start() {} + + @Override + public boolean flush() { + return true; + } + + @Override + public void close() {} + + @Override + public void incrementDropCounts(int spanCount) {} + } + + @Benchmark + public void benchmark(Blackhole blackhole) { + blackhole.consume(aggregator.publish(spans)); + } +} From ed38f18c4100ff1b1bde377d4c098dce17ad79f2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 13:19:21 -0400 Subject: [PATCH 5/5] Tighten SpanKindFilter encapsulation Make SpanKindFilter.kindMask and its constructor private now that DDSpan.isKind no longer needs direct field access -- it delegates to SpanKindFilter.matches(byte). The Builder.build() in the same outer class still constructs instances via the private constructor. Co-Authored-By: Claude Opus 4.7 (1M context) --- dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java | 2 +- .../src/main/java/datadog/trace/core/SpanKindFilter.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index ab074d8d4c8..4c438e1c915 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -960,7 +960,7 @@ public boolean isOutbound() { } public boolean isKind(SpanKindFilter filter) { - return (filter.kindMask & (1 << context.getSpanKindOrdinal())) != 0; + return filter.matches(context.getSpanKindOrdinal()); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java index 600e0d9ca47..9ac3fa9dc06 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/SpanKindFilter.java @@ -42,9 +42,9 @@ public static final Builder builder() { return new Builder(); } - final int kindMask; + private final int kindMask; - SpanKindFilter(int kindMask) { + private SpanKindFilter(int kindMask) { this.kindMask = kindMask; }