Add rate limit on Span decoration probe errors (#10529)

jpbempel · sarahchen6 · web-flow · commit eff2a4b0df14 · 2026-02-05T22:16:55.000Z
Add rate limit on Span decoration probe errors

if an evaluation error is generated by a span decoration probe we
are rate limiting it

Co-authored-by: sarah.chen &lt;sarah.chen@datadoghq.com&gt;
diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java
@@ -18,6 +18,7 @@
 import datadog.trace.bootstrap.debugger.MethodLocation;
 import datadog.trace.bootstrap.debugger.ProbeId;
 import datadog.trace.bootstrap.debugger.ProbeImplementation;
+import datadog.trace.bootstrap.debugger.ProbeRateLimiter;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.util.TagsHelper;
@@ -294,6 +295,10 @@ private void handleEvaluationErrors(SpanDecorationStatus status) {
     if (status.getErrors().isEmpty()) {
       return;
     }
+    boolean sampled = ProbeRateLimiter.tryProbe(id);
+    if (!sampled) {
+      return;
+    }
     Snapshot snapshot = new Snapshot(Thread.currentThread(), this, -1);
     snapshot.addEvaluationErrors(status.getErrors());
     DebuggerAgent.getSink().addSnapshot(snapshot);
diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java
@@ -41,6 +41,7 @@
 import datadog.trace.bootstrap.debugger.MethodLocation;
 import datadog.trace.bootstrap.debugger.ProbeId;
 import datadog.trace.bootstrap.debugger.ProbeImplementation;
+import datadog.trace.bootstrap.debugger.ProbeRateLimiter;
 import datadog.trace.bootstrap.debugger.util.Redaction;
 import datadog.trace.core.CoreTracer;
 import java.io.IOException;
@@ -78,6 +79,7 @@ public void setUp() {
   public void after() {
     super.after();
     Redaction.clearUserDefinedTypes();
+    ProbeRateLimiter.resetAll();
   }
 
   @Test
@@ -188,6 +190,11 @@ public void methodTagEvalError() throws IOException, URISyntaxException {
     assertNull(span.getTags().get("tag1"));
     assertEquals(
         "Cannot dereference field: noarg", span.getTags().get("_dd.di.tag1.evaluation_error"));
+    assertEquals(1, mockSink.getSnapshots().size());
+    Snapshot snapshot = mockSink.getSnapshots().get(0);
+    assertEquals(1, snapshot.getEvaluationErrors().size());
+    assertEquals(
+        "Cannot dereference field: noarg", snapshot.getEvaluationErrors().get(0).getMessage());
   }
 
   @Test
@@ -588,6 +595,43 @@ public void typeRedactionConditions() throws IOException, URISyntaxException {
         snapshot.getEvaluationErrors().get(2).getMessage());
   }
 
+  @Test
+  public void ensureCallingSamplingTagEvalError() throws IOException, URISyntaxException {
+    doSamplingTest(this::methodTagEvalError, 1, 1);
+  }
+
+  @Test
+  public void ensureCallingSamplingMethodInvalidCondition() throws IOException, URISyntaxException {
+    doSamplingTest(this::methodActiveSpanInvalidCondition, 1, 1);
+  }
+
+  @Test
+  public void ensureCallingSamplingLineInvalidCondition() throws IOException, URISyntaxException {
+    doSamplingTest(this::lineActiveSpanInvalidCondition, 1, 1);
+  }
+
+  @Test
+  public void ensureCallingSamplingKeywordRedactionConditions()
+      throws IOException, URISyntaxException {
+    doSamplingTest(this::keywordRedactionConditions, 1, 1);
+  }
+
+  private void doSamplingTest(
+      CapturingTestBase.TestMethod testRun, int expectedGlobalCount, int expectedProbeCount)
+      throws IOException, URISyntaxException {
+    MockSampler probeSampler = new MockSampler();
+    MockSampler globalSampler = new MockSampler();
+    ProbeRateLimiter.setSamplerSupplier(rate -> rate < 101 ? probeSampler : globalSampler);
+    ProbeRateLimiter.setGlobalSnapshotRate(1000);
+    try {
+      testRun.run();
+    } finally {
+      ProbeRateLimiter.setSamplerSupplier(null);
+    }
+    assertEquals(expectedGlobalCount, globalSampler.getCallCount());
+    assertEquals(expectedProbeCount, probeSampler.getCallCount());
+  }
+
   private SpanDecorationProbe.Decoration createDecoration(String tagName, String valueDsl) {
     List<SpanDecorationProbe.Tag> tags =
         asList(
