From 5617bbe308a56f0562f949d54a62bef542b601e7 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 13:51:46 -0400 Subject: [PATCH 1/7] Cap quarkus-native native-image builder heap to fix arm64 OOM. --- .../application/src/main/resources/application.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties index c913c3f19fe..d582f0efed5 100644 --- a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties +++ b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties @@ -2,3 +2,7 @@ quarkus.log.level=INFO quarkus.log.category."datadog.smoketest".level=DEBUG quarkus.log.console.format=%d %-5p [%c] '%t' |MT|%X{dd.trace_id}|MS|%X{dd.span_id}|%m%e%n +# Cap native-image builder heap so spring + quarkus native builds fit the CI limited container. +# Mirrors spring's -Xmx4096M. +quarkus.native.native-image-xmx=4g + From d8ec521ba50db7ad4a420f99d4b237a9b0ae8e24 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 16:56:31 -0400 Subject: [PATCH 2/7] Limit CPU usage on CI. --- .../quarkus-native/application/build.gradle | 24 +++++++++++++------ .../src/main/resources/application.properties | 4 ---- .../application/build.gradle | 10 ++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/dd-smoke-tests/quarkus-native/application/build.gradle b/dd-smoke-tests/quarkus-native/application/build.gradle index 8b9c83da46d..b43bcecc721 100644 --- a/dd-smoke-tests/quarkus-native/application/build.gradle +++ b/dd-smoke-tests/quarkus-native/application/build.gradle @@ -15,6 +15,12 @@ if (hasProperty('appBuildDir')) { version = "" +// Match native-image build threads to the CPUs the CI pod requested: the container sets a CPU +// request but no limit, so native-image would otherwise size to the whole host and its off-heap +// would exhaust memory. Falls back to 4 if the request is absent. +def isCI = providers.environmentVariable("CI").isPresent() +def ciBuildCpuCount = providers.environmentVariable("KUBERNETES_CPU_REQUEST").getOrElse("4") + dependencies { implementation enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}") implementation 'io.quarkus:quarkus-resteasy' @@ -36,11 +42,15 @@ if (hasProperty('agentJar')) { // are not properly tracked by this nested gradle build cache, it needs to be tracked // from the outside (see smokeTestApp). final agentJar = property('agentJar') - System.setProperty( - 'quarkus.native.additional-build-args', - "-J-javaagent:${agentJar}," + - "-J-Ddatadog.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd'T'HH:mm:ss.SSS'Z [dd.trace]'," + - "-J-Ddd.profiling.enabled=true," + - "-march=native" - ) + def nativeBuildArgs = [ + "-J-javaagent:${agentJar}", + "-J-Ddatadog.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd'T'HH:mm:ss.SSS'Z [dd.trace]'", + "-J-Ddd.profiling.enabled=true", + "-march=native", + ] + if (isCI) { + nativeBuildArgs.add("-H:NumberOfThreads=$ciBuildCpuCount") + nativeBuildArgs.add("-J-XX:ActiveProcessorCount=$ciBuildCpuCount") + } + System.setProperty('quarkus.native.additional-build-args', nativeBuildArgs.join(',')) } diff --git a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties index d582f0efed5..c913c3f19fe 100644 --- a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties +++ b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties @@ -2,7 +2,3 @@ quarkus.log.level=INFO quarkus.log.category."datadog.smoketest".level=DEBUG quarkus.log.console.format=%d %-5p [%c] '%t' |MT|%X{dd.trace_id}|MS|%X{dd.span_id}|%m%e%n -# Cap native-image builder heap so spring + quarkus native builds fit the CI limited container. -# Mirrors spring's -Xmx4096M. -quarkus.native.native-image-xmx=4g - diff --git a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle index 01eb6cecb8e..08c63e748d2 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle +++ b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle @@ -13,6 +13,12 @@ apply from: "$sharedConfigDirectory/repositories.gradle" ext.withProfiler = hasProperty('profiler') +// Match native-image build threads to the CPUs the CI pod requested: the container sets a CPU +// request but no limit, so native-image would otherwise size to the whole host and its off-heap +// would exhaust memory. Falls back to 4 if the request is absent. +def isCI = providers.environmentVariable("CI").isPresent() +def ciBuildCpuCount = providers.environmentVariable("KUBERNETES_CPU_REQUEST").getOrElse("4") + if (hasProperty('appBuildDir')) { buildDir = property('appBuildDir') } @@ -47,6 +53,10 @@ if (hasProperty('agentPath')) { buildArgs.add("-J-Ddd.profiling.scrub.enabled=true") buildArgs.add("-J-Ddd.profiling.start-force-first=true") } + if (isCI) { + buildArgs.add("-H:NumberOfThreads=$ciBuildCpuCount") + buildArgs.add("-J-XX:ActiveProcessorCount=$ciBuildCpuCount") + } jvmArgs.add("-Xmx4096M") } } From bfd7be5b341824feadb36ee5706afd938c1dda7d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 17:27:41 -0400 Subject: [PATCH 3/7] Cap Quarkus app. --- .../application/src/main/resources/application.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties index c913c3f19fe..a3d72060689 100644 --- a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties +++ b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties @@ -2,3 +2,7 @@ quarkus.log.level=INFO quarkus.log.category."datadog.smoketest".level=DEBUG quarkus.log.console.format=%d %-5p [%c] '%t' |MT|%X{dd.trace_id}|MS|%X{dd.span_id}|%m%e%n +# Cap the native-image builder heap so spring + quarkus native builds fit the CI limited container. +# Mirrors spring's -Xmx4096M. +quarkus.native.native-image-xmx=4g + From 92422efeb380058f1711472ce89ea55c0faa7c89 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 18:18:15 -0400 Subject: [PATCH 4/7] Use L tier for GraalVM to avoid OOM. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf1cb6d91b5..88cb505566d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1107,6 +1107,7 @@ test_smoke_graalvm: needs: *needs_build_tests_smoke tags: [ "arch:amd64" ] variables: + <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" @@ -1119,6 +1120,7 @@ test_smoke_graalvm_arm64: extends: .test_job_arm64 tags: [ "arch:arm64" ] variables: + <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" From 629f78fb483f5f4e2f4274fac099965be7b2af9f Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 20:10:59 -0400 Subject: [PATCH 5/7] Cap CPU to 6. --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88cb505566d..5d580a79428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1108,6 +1108,8 @@ test_smoke_graalvm: tags: [ "arch:amd64" ] variables: <<: *tier_l_variables + # Caps native-image to 6 build threads. + KUBERNETES_CPU_REQUEST: 6 GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" @@ -1121,6 +1123,8 @@ test_smoke_graalvm_arm64: tags: [ "arch:arm64" ] variables: <<: *tier_l_variables + # Caps native-image to 6 build threads. + KUBERNETES_CPU_REQUEST: 6 GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" From cae5599e419c9d1edf340d30a11d4b84f6de69ed Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 21:16:12 -0400 Subject: [PATCH 6/7] Removed 6 CPU cap. Added MASS for arm64. --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d580a79428..4b2b55f0d49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -785,6 +785,11 @@ muzzle-dep-report: - sudo chown -R 1001:1001 .gradle - export GRADLE_USER_HOME=$(pwd)/.gradle - sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties + # Route Gradle distribution download through MASS pull-through cache + - | + mass_read_host="${MASS_READ_URL#https://}" + mass_read_host="${mass_read_host%/}" + sed -i "/^distributionUrl=/ s|services.gradle.org|${mass_read_host}/internal/artifact/services.gradle.org|" gradle/wrapper/gradle-wrapper.properties - *normalize_node_index - *prepare_test_env # Disable CDS in forked JVMs to avoid SIGSEGVs on Linux arm64. @@ -1108,8 +1113,6 @@ test_smoke_graalvm: tags: [ "arch:amd64" ] variables: <<: *tier_l_variables - # Caps native-image to 6 build threads. - KUBERNETES_CPU_REQUEST: 6 GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" @@ -1123,8 +1126,6 @@ test_smoke_graalvm_arm64: tags: [ "arch:arm64" ] variables: <<: *tier_l_variables - # Caps native-image to 6 build threads. - KUBERNETES_CPU_REQUEST: 6 GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" From 5989920060e0ec3767c587b9704d80f91008e8df Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 22 Jun 2026 22:25:30 -0400 Subject: [PATCH 7/7] Tweaked comments. --- dd-smoke-tests/quarkus-native/application/build.gradle | 4 +--- .../application/src/main/resources/application.properties | 3 --- .../spring-boot-3.0-native/application/build.gradle | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/dd-smoke-tests/quarkus-native/application/build.gradle b/dd-smoke-tests/quarkus-native/application/build.gradle index b43bcecc721..65beb08e810 100644 --- a/dd-smoke-tests/quarkus-native/application/build.gradle +++ b/dd-smoke-tests/quarkus-native/application/build.gradle @@ -15,9 +15,7 @@ if (hasProperty('appBuildDir')) { version = "" -// Match native-image build threads to the CPUs the CI pod requested: the container sets a CPU -// request but no limit, so native-image would otherwise size to the whole host and its off-heap -// would exhaust memory. Falls back to 4 if the request is absent. +// Avoid unlimited CPU usage on CI. def isCI = providers.environmentVariable("CI").isPresent() def ciBuildCpuCount = providers.environmentVariable("KUBERNETES_CPU_REQUEST").getOrElse("4") diff --git a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties index a3d72060689..2e3c80d417d 100644 --- a/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties +++ b/dd-smoke-tests/quarkus-native/application/src/main/resources/application.properties @@ -1,8 +1,5 @@ quarkus.log.level=INFO quarkus.log.category."datadog.smoketest".level=DEBUG quarkus.log.console.format=%d %-5p [%c] '%t' |MT|%X{dd.trace_id}|MS|%X{dd.span_id}|%m%e%n - -# Cap the native-image builder heap so spring + quarkus native builds fit the CI limited container. -# Mirrors spring's -Xmx4096M. quarkus.native.native-image-xmx=4g diff --git a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle index 08c63e748d2..08545cf1e16 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle +++ b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle @@ -13,9 +13,7 @@ apply from: "$sharedConfigDirectory/repositories.gradle" ext.withProfiler = hasProperty('profiler') -// Match native-image build threads to the CPUs the CI pod requested: the container sets a CPU -// request but no limit, so native-image would otherwise size to the whole host and its off-heap -// would exhaust memory. Falls back to 4 if the request is absent. +// Avoid unlimited CPU usage on CI. def isCI = providers.environmentVariable("CI").isPresent() def ciBuildCpuCount = providers.environmentVariable("KUBERNETES_CPU_REQUEST").getOrElse("4")