Skip to content

Commit 5fb5676

Browse files
jbachorikjbachorik
committed
feat(profiling): Add JFR to OTLP/P converter with CLI, benchmarks and validation
New profiling-otel module: converts JFR recordings to OTLP Profiles protobufs — string/function/location/stack/link dictionary tables, hand-rolled protobuf encoder, jdk.* and jdk.CPUTimeSample event handling, deduplicating resource attributes. Ships a standalone CLI (convert-jfr.sh), JMH benchmark suite, profcheck-based validation harness with pinned OTLP proto schemas and an OTel collector testcontainer round-trip, plus module docs (ARCHITECTURE, BENCHMARKS, CLI, INTEGRATION_PLAN, PROFCHECK_INTEGRATION) and the Dockerfile.profcheck image. Also claims CODEOWNERS for the profcheck Dockerfile.
1 parent 318c6c6 commit 5fb5676

60 files changed

Lines changed: 10864 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@
361361

362362
# @DataDog/profiling-java
363363
/dd-java-agent/ddprof-lib/ @DataDog/profiling-java
364+
/docker/Dockerfile.profcheck @DataDog/profiling-java
364365
/dd-java-agent/agent-profiling/ @DataDog/profiling-java
365366
/dd-java-agent/agent-crashtracking/ @DataDog/profiling-java
366367
/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/jfr/ @DataDog/profiling-java

‎dd-java-agent/agent-profiling/profiling-otel/build.gradle.kts‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
plugins {
2+
`java-library`
3+
id("dd-trace-java.module.internal-library")
4+
id("com.gradleup.shadow")
5+
id("me.champeau.jmh")
6+
}
7+
8+
// Tests require Java 17+ (JMC 9.1.1); on JVM 8 (where -PcheckCoverage runs) tests are skipped,
9+
// so all main classes would have 0% coverage. Exclude them from verification.
10+
extra["excludedClassesCoverage"] = listOf(
11+
"com.datadog.profiling.otel.JfrToOtlpConverter*",
12+
"com.datadog.profiling.otel.JfrToOtlpConverterCLI",
13+
"com.datadog.profiling.otel.jfr.*",
14+
"com.datadog.profiling.otel.proto.*",
15+
"com.datadog.profiling.otel.proto.dictionary.*",
16+
)
17+
18+
jmh {
19+
jmhVersion = libs.versions.jmh.get()
20+
21+
// Allow filtering benchmarks via command line
22+
// Usage: ./gradlew jmh -PjmhIncludes="JfrToOtlpConverterBenchmark"
23+
// Usage: ./gradlew jmh -PjmhIncludes=".*convertJfrToOtlp"
24+
if (project.hasProperty("jmhIncludes")) {
25+
val pattern = project.property("jmhIncludes") as String
26+
includes = listOf(pattern)
27+
}
28+
29+
// Profiling support
30+
// Usage: ./gradlew jmh -PjmhProfile=true
31+
// Generates flamegraph and allocation profile
32+
if (project.hasProperty("jmhProfile")) {
33+
profilers = listOf("gc", "stack")
34+
jvmArgs = listOf(
35+
"-XX:+UnlockDiagnosticVMOptions",
36+
"-XX:+DebugNonSafepoints"
37+
)
38+
}
39+
40+
// async-profiler CPU flamegraph
41+
// Usage: ./gradlew jmh -PjmhAsyncProfiler=/path/to/libasyncProfiler.dylib
42+
// Output: /tmp/jmh-async-profile.html
43+
if (project.hasProperty("jmhAsyncProfiler")) {
44+
val lib = project.property("jmhAsyncProfiler") as String
45+
jvmArgs = listOf(
46+
"-XX:+UnlockDiagnosticVMOptions",
47+
"-XX:+DebugNonSafepoints",
48+
"-agentpath:$lib=start,event=cpu,file=/tmp/jmh-async-profile.html,flamegraph"
49+
)
50+
}
51+
}
52+
53+
// OTLP validation tests removed - use profcheck validation instead (see validateOtlp task below)
54+
55+
repositories {
56+
maven {
57+
url = uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")
58+
mavenContent {
59+
snapshotsOnly()
60+
// jafar is only published as a snapshot; restrict resolution to that group
61+
includeGroup("io.btrace")
62+
}
63+
}
64+
}
65+
66+
configure<datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension> {
67+
minJavaVersion = JavaVersion.VERSION_17
68+
}
69+
70+
tasks.named<JavaCompile>("compileTestJava") {
71+
// JMC 9.1.1 requires Java 17, and we need jdk.jfr.Event for stack trace testing
72+
options.release.set(17)
73+
javaCompiler.set(
74+
javaToolchains.compilerFor { languageVersion.set(JavaLanguageVersion.of(17)) }
75+
)
76+
}
77+
78+
tasks.named<JavaCompile>("compileJmhJava") {
79+
// JMC 9.1.1 requires Java 17, and we need jdk.jfr.Event for JMH benchmarks
80+
options.release.set(17)
81+
javaCompiler.set(
82+
javaToolchains.compilerFor { languageVersion.set(JavaLanguageVersion.of(17)) }
83+
)
84+
}
85+
86+
// Create fat jar for standalone CLI usage
87+
tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar") {
88+
archiveClassifier.set("cli")
89+
manifest {
90+
attributes["Main-Class"] = "com.datadog.profiling.otel.JfrToOtlpConverterCLI"
91+
}
92+
// Exclude SLF4J service provider files to avoid warnings
93+
exclude("META-INF/services/org.slf4j.spi.SLF4JServiceProvider")
94+
}
95+
96+
// CLI task for converting JFR files
97+
// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="input.jfr output.pb"
98+
// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="--json input.jfr output.json"
99+
tasks.register<JavaExec>("convertJfr") {
100+
group = "application"
101+
description = "Convert JFR recording to OTLP profiles format"
102+
classpath = sourceSets["main"].runtimeClasspath
103+
mainClass.set("com.datadog.profiling.otel.JfrToOtlpConverterCLI")
104+
105+
// Uses Gradle's built-in --args parameter which properly handles spaces in paths
106+
}
107+
108+
// Build profcheck Docker image
109+
// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:buildProfcheck
110+
tasks.register<Exec>("buildProfcheck") {
111+
group = "verification"
112+
description = "Build profcheck Docker image for OTLP validation"
113+
workingDir = rootDir
114+
commandLine("docker", "build", "-f", "docker/Dockerfile.profcheck", "-t", "profcheck:latest", ".")
115+
116+
// Check if Docker is available
117+
doFirst {
118+
val process = ProcessBuilder("docker", "info").redirectErrorStream(true).start()
119+
if (process.waitFor() != 0) {
120+
throw org.gradle.api.GradleException("Docker is not available. Profcheck validation requires Docker to be running.")
121+
}
122+
}
123+
}
124+
125+
// OTLP validation tests (ProfcheckValidationTest, OtlpCollectorValidationTest) are gated behind
126+
// the -PrunOtlpValidation Gradle property. They require Docker and network access to pull/build
127+
// images, which is not available in the standard CI test matrix. Enable manually:
128+
// ./gradlew :dd-java-agent:agent-profiling:profiling-otel:test -PrunOtlpValidation
129+
tasks.named<Test>("test") {
130+
if (project.hasProperty("runOtlpValidation")) {
131+
jvmArgs("-Drun.otlp.validation=true")
132+
133+
// Build profcheck image if Docker is available (for ProfcheckValidationTest)
134+
doFirst {
135+
val dockerAvailable = try {
136+
val process = ProcessBuilder("docker", "info").redirectErrorStream(true).start()
137+
process.waitFor() == 0
138+
} catch (e: Exception) {
139+
false
140+
}
141+
142+
if (dockerAvailable) {
143+
logger.lifecycle("Building profcheck Docker image for validation tests...")
144+
val buildProcess = ProcessBuilder(
145+
"docker",
146+
"build",
147+
"-f",
148+
"$rootDir/docker/Dockerfile.profcheck",
149+
"-t",
150+
"profcheck:latest",
151+
rootDir.toString()
152+
).redirectErrorStream(true).start()
153+
// drain the output while waiting — docker build routinely exceeds the OS pipe-buffer
154+
// capacity (~64 KiB) and an unread full pipe deadlocks waitFor()
155+
val outputBuilder = StringBuilder()
156+
val drainThread = Thread {
157+
buildProcess.inputStream.bufferedReader().forEachLine { line ->
158+
synchronized(outputBuilder) { outputBuilder.appendLine(line) }
159+
}
160+
}
161+
drainThread.isDaemon = true
162+
drainThread.start()
163+
val exitCode = buildProcess.waitFor()
164+
drainThread.join(10_000)
165+
if (exitCode != 0) {
166+
val output = synchronized(outputBuilder) { outputBuilder.toString() }
167+
throw org.gradle.api.GradleException("Failed to build profcheck Docker image (exit $exitCode):\n$output")
168+
}
169+
} else {
170+
// without the image the docker-tagged validation tests cannot run
171+
logger.warn("Docker not available: excluding tests tagged 'docker'.")
172+
useJUnitPlatform {
173+
excludeTags("docker")
174+
}
175+
}
176+
}
177+
}
178+
}
179+
180+
// Validate OTLP output using profcheck
181+
// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:validateOtlp -PotlpFile=/path/to/output.pb
182+
tasks.register<Exec>("validateOtlp") {
183+
group = "verification"
184+
description = "Validate OTLP profile using profcheck (requires Docker)"
185+
186+
// Ensure profcheck image exists
187+
dependsOn("buildProfcheck")
188+
189+
doFirst {
190+
if (!project.hasProperty("otlpFile")) {
191+
throw org.gradle.api.GradleException("Property 'otlpFile' is required. Usage: -PotlpFile=/path/to/output.pb")
192+
}
193+
194+
val otlpFilePath = project.property("otlpFile") as String
195+
val otlpFile = file(otlpFilePath)
196+
197+
if (!otlpFile.exists()) {
198+
throw org.gradle.api.GradleException("File not found: $otlpFilePath")
199+
}
200+
201+
val parentDir = otlpFile.parentFile.absolutePath
202+
val fileName = otlpFile.name
203+
204+
// Run profcheck in Docker with volume mount
205+
commandLine(
206+
"docker",
207+
"run",
208+
"--rm",
209+
"-v",
210+
"$parentDir:/data:ro",
211+
"profcheck:latest",
212+
"/data/$fileName"
213+
)
214+
}
215+
}
216+
217+
dependencies {
218+
implementation(libs.jafar.parser)
219+
implementation(project(":internal-api"))
220+
implementation(project(":components:json"))
221+
222+
testImplementation(libs.bundles.junit5)
223+
testImplementation(libs.bundles.jmc)
224+
testImplementation(libs.jmc.flightrecorder.writer)
225+
testImplementation(libs.testcontainers)
226+
testImplementation("org.testcontainers:junit-jupiter:1.21.3")
227+
testImplementation(libs.testing.okhttp3)
228+
}

0 commit comments

Comments
 (0)