Skip to content

Commit 8026a64

Browse files
jbachorikjbachorik
committed
Add reference-chain repro, sweep, and reporting tooling
1 parent 99d1e7a commit 8026a64

15 files changed

Lines changed: 1804 additions & 19 deletions

‎.claude/commands/build-and-summarize‎

Lines changed: 160 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,164 @@ else
6767
echo "✖ Gradle failed with status $status. Full log at: $LOG"
6868
fi
6969

70-
# Hand over to your logs analyst agent — keep the main session output tiny.
70+
# Generate the summary artifacts deterministically — no LLM, no claude spawn.
71+
# (The old tail spawned a headless `claude` per build; managed settings pin that
72+
# to sonnet-5 via the AI gateway, billing real money for every log parse.)
7173
echo
72-
echo "Delegating to gradle-logs-analyst agent…"
73-
# If your CLI supports non-streaming, set it here to avoid verbose output.
74-
# Example (uncomment if supported): export CLAUDE_NO_STREAM=1
75-
76-
# Sub-agents would inherit the full parent env, including CLAUDECODE, which
77-
# apparently causes problems with some versions of Claude (e.g. 4.6).
78-
unset CLAUDECODE
79-
claude "Act as the gradle-logs-analyst agent to parse the build log at: $LOG. Generate the required Gradle summary artifacts as specified in the gradle-logs-analyst agent definition."
74+
echo "Generating gradle summary…"
75+
python3 - "$LOG" <<'PYEOF' || echo "⚠ gradle summary generation failed — full log at: $LOG"
76+
import json, os, re, sys
77+
78+
log_path = sys.argv[1] if len(sys.argv) > 1 else ""
79+
OUT_DIR = os.path.join("build", "reports", "claude")
80+
MD = os.path.join(OUT_DIR, "gradle-summary.md")
81+
JS = os.path.join(OUT_DIR, "gradle-summary.json")
82+
83+
status, total_time, cur_task = "UNKNOWN", None, None
84+
failed_tasks, headlines, dep_issues, test_failures = [], [], [], []
85+
warnings, seen_warnings = [], set()
86+
tests = {"total": 0, "failed": 0, "skipped": 0, "modules": {}}
87+
88+
RE_STATUS = re.compile(r"^BUILD (SUCCESSFUL|FAILED)(?: in (.+?))?\s*$")
89+
RE_TASK = re.compile(r"^> Task (\S+)")
90+
RE_GSUM = re.compile(r"(\d+) tests? (successful|failed|skipped)")
91+
RE_COMP = re.compile(r"(\d+) tests? completed(?:, (\d+) failed)?(?:, (\d+) skipped)?")
92+
RE_GTEST_P = re.compile(r"\[ PASSED \] (\d+) tests?")
93+
RE_GTEST_F = re.compile(r"\[ FAILED \] (\d+) tests?")
94+
RE_DEP = re.compile(r"Could not (?:resolve|find|get)|timed out|status code: 40[13]|artifact .* not found", re.I)
95+
RE_WARN = re.compile(r"\bw: |warning:|deprecat", re.I)
96+
97+
def mod(name):
98+
return tests["modules"].setdefault(name, {"total": 0, "failed": 0, "skipped": 0})
99+
100+
def add_counts(name, ok, failed, skipped):
101+
m = mod(name)
102+
m["total"] += ok + failed + skipped
103+
m["failed"] += failed
104+
m["skipped"] += skipped
105+
tests["total"] += ok + failed + skipped
106+
tests["failed"] += failed
107+
tests["skipped"] += skipped
108+
109+
if not log_path or not os.path.isfile(log_path) or os.path.getsize(log_path) == 0:
110+
why = f"log not found: {log_path!r}" if not log_path or not os.path.isfile(log_path) else "log is empty"
111+
os.makedirs(OUT_DIR, exist_ok=True)
112+
open(MD, "w").write(f"# Gradle Summary\n\n- Status: UNKNOWN ({why})\n\nFull log unavailable or empty.\n")
113+
json.dump({"status": "UNKNOWN", "totalTime": None, "failedTasks": [], "warnings": [],
114+
"tests": tests, "slowTasks": [], "depIssues": [why], "actions": []}, open(JS, "w"), indent=1)
115+
print(f"Gradle log unusable ({why}); wrote {MD} and {JS}")
116+
sys.exit(0)
117+
118+
grab_headline = False
119+
with open(log_path, encoding="utf-8", errors="replace") as f:
120+
for line in f:
121+
line = line.rstrip("\n")
122+
m = RE_STATUS.match(line)
123+
if m:
124+
status = m.group(1)
125+
total_time = m.group(2)
126+
continue
127+
m = RE_TASK.match(line)
128+
if m:
129+
cur_task = m.group(1)
130+
if line.endswith("FAILED") and m.group(1) not in failed_tasks:
131+
failed_tasks.append(m.group(1))
132+
continue
133+
if "FAILURE: Build failed with an exception" in line:
134+
grab_headline = True
135+
continue
136+
if grab_headline and line.strip().startswith("* What went wrong"):
137+
headlines.append("")
138+
continue
139+
if grab_headline:
140+
s = line.strip()
141+
if s.startswith("*") or not s:
142+
if headlines and headlines[-1]:
143+
grab_headline = False
144+
continue
145+
if len(headlines[-1]) < 200:
146+
headlines[-1] += (" " if headlines[-1] else "") + s
147+
continue
148+
m = RE_COMP.search(line)
149+
if m:
150+
total = int(m.group(1))
151+
failed = int(m.group(2) or 0)
152+
skipped = int(m.group(3) or 0)
153+
add_counts(cur_task or "(unknown)", total - failed - skipped, failed, skipped)
154+
continue
155+
m = RE_GSUM.search(line)
156+
if m:
157+
n, verb = int(m.group(1)), m.group(2)
158+
mm = mod(cur_task or "(unknown)")
159+
mm["total"] += n
160+
tests["total"] += n
161+
if verb == "failed":
162+
mm["failed"] += n
163+
tests["failed"] += n
164+
elif verb == "skipped":
165+
mm["skipped"] += n
166+
tests["skipped"] += n
167+
continue
168+
m = RE_GTEST_P.search(line)
169+
if m:
170+
add_counts(cur_task or "(gtest)", int(m.group(1)), 0, 0)
171+
continue
172+
m = RE_GTEST_F.search(line)
173+
if m:
174+
add_counts(cur_task or "(gtest)", 0, int(m.group(1)), 0)
175+
continue
176+
if re.search(r" > \S+(?:\.\S+)+.* FAILED$", line) or re.search(r"\[ FAILED \] (\S+)", line):
177+
test_failures.append(line.strip())
178+
continue
179+
if RE_DEP.search(line) and len(dep_issues) < 20:
180+
if line not in dep_issues:
181+
dep_issues.append(line.strip())
182+
continue
183+
wm = RE_WARN.search(line)
184+
if wm and len(warnings) < 40:
185+
key = line.strip()[:160]
186+
if key not in seen_warnings:
187+
seen_warnings.add(key)
188+
warnings.append(key)
189+
190+
failed = [t + (" — " + h if h else "") for t, h in zip(failed_tasks, headlines + [""] * len(failed_tasks))]
191+
data = {"status": status, "totalTime": total_time, "failedTasks": failed, "warnings": warnings,
192+
"tests": tests, "slowTasks": [], "depIssues": dep_issues, "actions": []}
193+
194+
os.makedirs(OUT_DIR, exist_ok=True)
195+
with open(JS, "w") as f:
196+
json.dump(data, f, indent=1)
197+
198+
L = ["# Gradle Summary\n", f"- Status: {status}", f"- Total time: {total_time or 'UNKNOWN'}",
199+
f"- Log: {log_path}\n"]
200+
if failed:
201+
L.append("## Failing Tasks")
202+
L += [f"- {t}" for t in failed]
203+
L.append("")
204+
if headlines:
205+
L.append("## Primary Failure")
206+
L += [f"```text\n{h}\n```" for h in headlines[:3] if h]
207+
L.append("")
208+
if tests["total"]:
209+
L.append("## Tests")
210+
L.append(f"- Total: {tests['total']}, failed: {tests['failed']}, skipped: {tests['skipped']}")
211+
for name, m in tests["modules"].items():
212+
L.append(f" - {name}: {m['total']} total, {m['failed']} failed, {m['skipped']} skipped")
213+
if test_failures:
214+
L.append("- Top failing tests:")
215+
L += [f" - {t}" for t in test_failures[:10]]
216+
L.append("")
217+
if warnings:
218+
L.append("## Warnings")
219+
L += [f"- {w}" for w in warnings[:20]]
220+
L.append("")
221+
if dep_issues:
222+
L.append("## Dependency / Network Issues")
223+
L += [f"- {d}" for d in dep_issues[:10]]
224+
L.append("")
225+
L.append("- Per-task durations are not present in plain `-i` console logs; slowTasks omitted.")
226+
open(MD, "w").write("\n".join(L) + "\n")
227+
228+
print(f"{status} (time: {total_time or 'n/a'}); {len(failed)} failing task(s), {tests['failed']} test failure(s)")
229+
print(f"Wrote {MD} and {JS}")
230+
PYEOF

‎.gitignore‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,17 @@ doc/temp/
4444
# CLAUDE.md is auto-generated from AGENTS.md bootstrap instructions
4545
CLAUDE.md
4646

47-
.sphinx
47+
# OS/editor cruft
48+
.DS_Store
49+
50+
# AI review/agent tooling scratch state
51+
.sphinx/
52+
.skill-builder-temp/
53+
.claude/scheduled_tasks.lock
54+
55+
# Python bytecode cache
56+
__pycache__/
57+
*.pyc
58+
59+
# AI investigation scratch state (never committed)
60+
.investigations/

‎ddprof-stresstest/build.gradle.kts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,45 @@ tasks.register<Jar>("chaosJar") {
101101
}
102102
}
103103

104+
// --- reference-chains repro app ---------------------------------------------
105+
// Plain, dependency-free demo app for manually reproducing the reference-chains
106+
// feature: launched directly with `-agentpath:libjavaProfiler.so=start,...`
107+
// (no dd-trace-java agent, no dynamic attach), so it deliberately has zero
108+
// compile/runtime dependency on ddprof-lib or dd-trace - the profiler is
109+
// entirely out-of-process from this app's own point of view.
110+
111+
sourceSets {
112+
create("repro")
113+
}
114+
115+
dependencies {
116+
// ddprof-lib's Java API only - not the native library itself (that's the one
117+
// already loaded in-process via -agentpath). Used solely to call
118+
// JavaProfiler.getInstance().dump(...) periodically: Profiler::dump()
119+
// (profiler.cpp) is the only code path that drains ReferenceChainTracker's
120+
// pending chain-event queue and actually writes datadog.ReferenceChain into
121+
// the JFR file - nothing does this automatically on a timer, and in
122+
// production it's dd-trace-java's own recording-chunk rotation that calls
123+
// it. Without this dependency+call, chain events get built and enqueued
124+
// (visible in TEST_LOG output) but are silently dropped when the process
125+
// exits, and the JFR file never shows a single datadog.ReferenceChain event.
126+
"reproImplementation"(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug")))
127+
}
128+
129+
tasks.register<Jar>("reproJar") {
130+
group = "build"
131+
description = "Demo app that leaks memory in a controlled way, for manually reproducing reference-chains via -agentpath"
132+
archiveFileName.set("refchains_repro.jar")
133+
from(sourceSets["repro"].output)
134+
from({
135+
configurations["reproRuntimeClasspath"].map { if (it.isDirectory) it else zipTree(it) }
136+
})
137+
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
138+
manifest {
139+
attributes("Main-Class" to "com.datadoghq.profiler.repro.ReferenceChainLeakDemo")
140+
}
141+
}
142+
104143
tasks.register<Exec>("runStressTests") {
105144
dependsOn(tasks.named("jmhJar"))
106145

‎ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ private static Antagonist create(String name) {
9292
return new WeakRefWaveAntagonist();
9393
case "dump-storm":
9494
return new DumpStormAntagonist();
95+
case "reference-chain-leak":
96+
return new ReferenceChainLeakAntagonist();
9597
case "reapply-context-value":
9698
return new ReapplyContextValueAntagonist();
9799
// Deferred: dlopen-churn (needs per-arch dummy .so built in CI prep).

‎ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReferenceChainLeakAntagonist.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright 2026, Datadog, Inc
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
package com.datadoghq.profiler.chaos;
11+
12+
import java.time.Duration;
13+
import java.util.Map;
14+
import java.util.concurrent.ConcurrentHashMap;
15+
16+
/**
17+
* Grows an ever-referenced, never-evicted cache to give the reference-chains
18+
* feature (LivenessTracker/ReferenceChainTracker) a real, steadily growing
19+
* population to detect and walk. Growth is capped, not open-ended: once
20+
* {@link #CAP_ENTRIES} is reached the cache stops growing and the antagonist
21+
* just holds it there for the rest of the run, so a long chaos duration
22+
* (hours) still produces the same bounded leak instead of eventually OOMing
23+
* the JVM out from under the other antagonists sharing its heap.
24+
*
25+
* <p>Same object shape as {@code LeakingCacheScenario.CachedPayload}
26+
* (ddprof-test's reference-chain integration test) — plain long fields
27+
* rather than a nested array, so a single instance clears the allocation
28+
* sampler's real size floor without a second, competing heap allocation per
29+
* entry.
30+
*/
31+
public final class ReferenceChainLeakAntagonist implements Antagonist {
32+
33+
// ~64MiB of CachedEntry payloads at steady state (entries are a little
34+
// over 300 bytes with object header/field overhead) — enough for the
35+
// liveness tracker's population-growth trend detection to have a real,
36+
// sustained trend to lock onto, without meaningfully competing with the
37+
// other antagonists' own memory budget for the run's whole duration.
38+
private static final int CAP_ENTRIES = 200_000;
39+
private static final int BATCH_SIZE = 200;
40+
private static final long BATCH_INTERVAL_MS = 50L;
41+
42+
private final Map<String, CachedEntry> cache = new ConcurrentHashMap<>();
43+
private volatile boolean running;
44+
private Thread growthDriver;
45+
private long nextKey;
46+
47+
@Override
48+
public String name() {
49+
return "reference-chain-leak";
50+
}
51+
52+
@Override
53+
public void start() {
54+
running = true;
55+
growthDriver = new Thread(this::growthLoop, "chaos-refchain-leak");
56+
growthDriver.setDaemon(true);
57+
growthDriver.start();
58+
}
59+
60+
@Override
61+
public void stopGracefully(Duration timeout) {
62+
running = false;
63+
try {
64+
growthDriver.join(timeout.toMillis());
65+
} catch (InterruptedException e) {
66+
Thread.currentThread().interrupt();
67+
}
68+
}
69+
70+
private void growthLoop() {
71+
while (running && cache.size() < CAP_ENTRIES) {
72+
for (int i = 0; i < BATCH_SIZE && cache.size() < CAP_ENTRIES; i++) {
73+
String key = "refchain-leak-" + (nextKey++);
74+
cache.put(key, new CachedEntry(key));
75+
MemoryGovernor.pace();
76+
}
77+
try {
78+
Thread.sleep(BATCH_INTERVAL_MS);
79+
} catch (InterruptedException e) {
80+
Thread.currentThread().interrupt();
81+
return;
82+
}
83+
}
84+
// Cap reached: keep the reference alive (do nothing) for the rest of
85+
// the run instead of exiting the thread, so `cache` stays reachable
86+
// via this antagonist's own field for as long as the harness runs.
87+
while (running) {
88+
try {
89+
Thread.sleep(1_000L);
90+
} catch (InterruptedException e) {
91+
Thread.currentThread().interrupt();
92+
return;
93+
}
94+
}
95+
}
96+
97+
static final class CachedEntry {
98+
final String key;
99+
long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15;
100+
long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31;
101+
102+
CachedEntry(String key) {
103+
this.key = key;
104+
}
105+
}
106+
}

0 commit comments

Comments
 (0)