Skip to content

Commit 47a088b

Browse files
sezen-datadogsezen-datadog
authored
Merge branch 'master' into sezen.leblay/upgrade-libddwaf-java-15.0.0
2 parents 7a77d7b + becbf2a commit 47a088b

26 files changed

Lines changed: 251 additions & 24 deletions

File tree

‎.gitlab-ci.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ variables:
3030
GRADLE_PLUGIN_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/gradle-plugin-portal-proxy/"
3131
BUILDER_IMAGE_VERSION_PREFIX: "v25.06-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
3232
REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
33-
DEFAULT_TEST_JVMS: /^(8|11|17|21)$/
33+
DEFAULT_TEST_JVMS: /^(8|11|17|21|stable)$/
3434
PROFILE_TESTS:
3535
description: "Enable profiling of tests"
3636
value: "false"
@@ -63,6 +63,7 @@ workflow:
6363
- "11"
6464
- "17"
6565
- "21"
66+
- "stable"
6667
- "semeru11"
6768
- "oracle8"
6869
- "zulu8"
@@ -560,7 +561,7 @@ test_inst_latest:
560561
CACHE_TYPE: "latestDep"
561562
parallel:
562563
matrix:
563-
- testJvm: ["8", "17", "21" ]
564+
- testJvm: ["8", "17", "21", "stable"]
564565
# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
565566
# This emulates "parallel" by including it in the matrix
566567
CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
@@ -612,7 +613,7 @@ test_debugger:
612613
variables:
613614
GRADLE_TARGET: ":debuggerTest"
614615
CACHE_TYPE: "base"
615-
DEFAULT_TEST_JVMS: /^(8|11|17|21|semeru8)$/
616+
DEFAULT_TEST_JVMS: /^(8|11|17|21|stable|semeru8)$/
616617
parallel:
617618
matrix: *test_matrix
618619

‎.gitlab/benchmarks.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,36 @@ benchmarks-post-results:
6868
- job: benchmarks-dacapo
6969
artifacts: true
7070

71+
check-big-regressions:
72+
extends: .benchmarks
73+
needs:
74+
- job: benchmarks-startup
75+
artifacts: true
76+
- job: benchmarks-load
77+
artifacts: true
78+
when: on_success
79+
tags: ["arch:amd64"]
80+
# ARTIFACTS_DIR /go/src/github.com/DataDog/apm-reliability/dd-trace-java/reports/
81+
# need to convert them
82+
script:
83+
- !reference [ .benchmarks, script ]
84+
- |
85+
for benchmarkType in startup load; do
86+
find "$ARTIFACTS_DIR/$benchmarkType" -name "benchmark-baseline.json" -o -name "benchmark-candidate.json" | while read file; do
87+
relpath="${file#$ARTIFACTS_DIR/$benchmarkType/}"
88+
prefix="${relpath%/benchmark-*}" # Remove the trailing /benchmark-(baseline|candidate).json
89+
prefix="${prefix#./}" # Remove any leading ./
90+
prefix="${prefix//\//-}" # Replace / with -
91+
case "$file" in
92+
*benchmark-baseline.json) type="baseline" ;;
93+
*benchmark-candidate.json) type="candidate" ;;
94+
esac
95+
echo "Moving $file to $ARTIFACTS_DIR/${type}-${prefix}.converted.json"
96+
cp "$file" "$ARTIFACTS_DIR/${type}-${prefix}.converted.json"
97+
done
98+
done
99+
- bp-runner $CI_PROJECT_DIR/.gitlab/benchmarks/bp-runner.fail-on-regression.yml --debug
100+
71101
.dsm-kafka-benchmarks:
72102
stage: benchmarks
73103
rules:

‎.gitlab/benchmarks/bp-runner.fail-on-regression.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
experiments:
2+
- name: Run regression check
3+
steps:
4+
- name: Regression Check
5+
run: fail_on_regression
6+
# Applies on all scenarios
7+
regression_threshold: 10.0 # percents

‎BUILDING.md‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ or on Windows:
2828
Your output should look something like the following:
2929

3030
```
31-
ℹ️ Checking required JVM:
32-
✅ JAVA_HOME is set to /Users/datadog/.sdkman/candidates/java/8.0.402-zulu.
33-
✅ JAVA_8_HOME is set to /Users/datadog/.sdkman/candidates/java/8.0.402-zulu.
34-
✅ JAVA_11_HOME is set to /Users/datadog/.sdkman/candidates/java/11.0.22-zulu.
35-
✅ JAVA_17_HOME is set to /Users/datadog/.sdkman/candidates/java/17.0.10-zulu.
36-
✅ JAVA_21_HOME is set to /Users/datadog/.sdkman/candidates/java/21.0.2-zulu.
37-
✅ JAVA_GRAALVM17_HOME is set to /Users/datadog/.sdkman/candidates/java/17.0.9-graalce.
31+
ℹ️ Checking required JVMs:
32+
✅ JAVA_HOME is set to /Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home.
33+
✅ JAVA_8_HOME is set to /Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home.
34+
✅ JAVA_11_HOME is set to /Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home.
35+
✅ JAVA_17_HOME is set to /Library/Java/JavaVirtualMachines/zulu-17.jdk/Contents/Home.
36+
✅ JAVA_21_HOME is set to /Library/Java/JavaVirtualMachines/zulu-21.jdk/Contents/Home.
37+
✅ JAVA_24_HOME is set to /Library/Java/JavaVirtualMachines/zulu-24.jdk/Contents/Home.
38+
✅ JAVA_GRAALVM17_HOME is set to /Library/Java/JavaVirtualMachines/graalvm-ce-java17-22.3.1/Contents/Home.
3839
ℹ️ Checking git configuration:
3940
✅ The git command line is installed.
4041
✅ pre-commit hook is installed in repository.
@@ -51,22 +52,22 @@ If there is any issue with your output, check the requirements above and use the
5152

5253
Requirements to build the full project:
5354

54-
* The JDK versions 8, 11, 17 and 21 must be installed.
55-
* The `JAVA_8_HOME`, `JAVA_11_HOME`, `JAVA_17_HOME`, `JAVA_21_HOME`, and `JAVA_GRAALVM17_HOME` must point to their respective JDK location.
55+
* The JDK versions 8, 11, 17, 21, and 24 must be installed.
56+
* The `JAVA_8_HOME`, `JAVA_11_HOME`, `JAVA_17_HOME`, `JAVA_21_HOME`, `JAVA_24_HOME`, and `JAVA_GRAALVM17_HOME` must point to their respective JDK location.
5657
* The JDK 8 `bin` directory must be the only JDK on the PATH (e.g. `$JAVA_8_HOME/bin`).
5758
* The `JAVA_HOME` environment variable may be unset. If set, it must point to the JDK 8 location (same as `JAVA_8_HOME`).
5859
* The `git` command line must be installed.
5960
* A container runtime environment must be available to run all tests (e.g. Docker Desktop).
6061

6162
### Install the required JDKs
6263

63-
Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
64+
Download and install JDK versions 8, 11, 17, 21 and 24, and GraalVM 17 for your OS.
6465

6566
#### macOS
6667

6768
* Install the required JDKs using `brew`:
6869
```shell
69-
brew install --cask zulu@8 zulu@11 zulu@17 zulu@21 graalvm/tap/graalvm-ce-java17
70+
brew install --cask zulu@8 zulu@11 zulu@17 zulu@21 zulu graalvm/tap/graalvm-ce-java17
7071
```
7172
* Fix the GraalVM installation by [removing the quarantine flag](https://www.graalvm.org/latest/docs/getting-started/macos/):
7273
```
@@ -78,6 +79,7 @@ Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
7879
export JAVA_11_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home
7980
export JAVA_17_HOME=/Library/Java/JavaVirtualMachines/zulu-17.jdk/Contents/Home
8081
export JAVA_21_HOME=/Library/Java/JavaVirtualMachines/zulu-21.jdk/Contents/Home
82+
export JAVA_24_HOME=/Library/Java/JavaVirtualMachines/zulu-24.jdk/Contents/Home
8183
export JAVA_GRAALVM17_HOME=/Library/Java/JavaVirtualMachines/graalvm-<current version of graalvm>/Contents/Home
8284
export JAVA_HOME=$JAVA_8_HOME
8385
```
@@ -93,22 +95,23 @@ Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
9395
9496
#### Linux
9597

96-
* Download and extract JDK 8, 11, 17, and 21 from [Eclipse Temurin releases](https://adoptium.net/temurin/releases/) and GraalVM 17 from [Oracle downloads](https://www.graalvm.org/downloads/).
98+
* Download and extract JDK 8, 11, 17, 21, and 24 from [Eclipse Temurin releases](https://adoptium.net/temurin/releases/) and GraalVM 17 from [Oracle downloads](https://www.graalvm.org/downloads/).
9799
* Install the GraalVM native image requirements for native builds by following [the GraalVM official documentation](https://www.graalvm.org/latest/reference-manual/native-image/#prerequisites).
98100
* Add the required environment variables to your shell using the `export` command. You can permanently install the environment variables by appending the `export` commands into your shell configuration file `~/.zshrc` or `~/.bashrc` or other.
99101
```shell
100102
export JAVA_8_HOME=/<path to extracted archive>/jdk8u<current version of JDK 8>
101103
export JAVA_11_HOME=/<path to extracted archive>/jdk-11.<current version of JDK 11>
102104
export JAVA_17_HOME=/<path to extracted archive>/jdk-17.<current version of JDK 17>
103105
export JAVA_21_HOME=/<path to extracted archive>/jdk-21.<current version of JDK 21>
106+
export JAVA_24_HOME=/<path to extracted archive>/jdk-24.<current version of JDK 24>
104107
export JAVA_GRAALVM17_HOME=/<path to extracted archive>/graalvm-jdk-17.<current version of graalvm>/Contents/Home
105108
export JAVA_HOME=$JAVA_8_HOME
106109
```
107110
* Restart your shell after applying the changes if you appended the commands to your shell configuration file.
108111

109112
#### Windows
110113

111-
* Download and install JDK 8, 11, 17, and 21 [Eclipse Temurin releases](https://adoptium.net/temurin/releases/).
114+
* Download and install JDK 8, 11, 17, 21, and 24 [Eclipse Temurin releases](https://adoptium.net/temurin/releases/).
112115

113116
<details>
114117
<summary>Alternatively, install JDKs using winget or scoop. (click here to expand)</summary>
@@ -118,6 +121,7 @@ Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
118121
winget install --id EclipseAdoptium.Temurin.11.JDK
119122
winget install --id EclipseAdoptium.Temurin.17.JDK
120123
winget install --id EclipseAdoptium.Temurin.21.JDK
124+
winget install --id EclipseAdoptium.Temurin.24.JDK
121125
```
122126
123127
```pwsh
@@ -126,6 +130,7 @@ Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
126130
scoop install temurin11-jdk
127131
scoop install temurin17-jdk
128132
scoop install temurin21-jdk
133+
scoop install temurin24-jdk
129134
```
130135

131136
</details>
@@ -136,6 +141,7 @@ Download and install JDK versions 8, 11, 17 and 21, and GraalVM 17 for your OS.
136141
[Environment]::SetEnvironmentVariable("JAVA_11_HOME", "C:\Program Files\Eclipse Adoptium\jdk-11.0.25.9-hotspot", [EnvironmentVariableTarget]::User)
137142
[Environment]::SetEnvironmentVariable("JAVA_17_HOME", "C:\Program Files\Eclipse Adoptium\jdk-17.0.12.7-hotspot", [EnvironmentVariableTarget]::User)
138143
[Environment]::SetEnvironmentVariable("JAVA_21_HOME", "C:\Program Files\Eclipse Adoptium\jdk-21.0.5.11-hotspot", [EnvironmentVariableTarget]::User)
144+
[Environment]::SetEnvironmentVariable("JAVA_24_HOME", "C:\Program Files\Eclipse Adoptium\jdk-24.0.1.9-hotspot", [EnvironmentVariableTarget]::User)
139145
140146
# JAVA_HOME = JAVA_8_HOME
141147
[Environment]::SetEnvironmentVariable("JAVA_HOME", "C:\Program Files\Eclipse Adoptium\jdk-8.0.432.6-hotspot", [EnvironmentVariableTarget]::User)

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import datadog.trace.api.ProductActivation;
4949
import datadog.trace.api.UserIdCollectionMode;
5050
import datadog.trace.api.telemetry.LogCollector;
51+
import datadog.trace.api.telemetry.WafMetricCollector;
5152
import java.io.ByteArrayInputStream;
5253
import java.io.FileInputStream;
5354
import java.io.FileNotFoundException;
@@ -263,7 +264,9 @@ private void handleWafUpdateResultReport(String configKey, Map<String, Object> r
263264
"Invalid rule during waf config update for config key {}: {}",
264265
configKey,
265266
e.wafDiagnostics);
266-
267+
if (e.wafDiagnostics.getNumConfigError() > 0) {
268+
WafMetricCollector.get().addWafConfigError(e.wafDiagnostics.getNumConfigError());
269+
}
267270
// TODO: Propagate diagostics back to remote config apply_error
268271

269272
initReporter.setReportForPublication(e.wafDiagnostics);

‎dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import datadog.remoteconfig.state.ProductListener
1313
import datadog.trace.api.Config
1414
import datadog.trace.api.ProductActivation
1515
import datadog.trace.api.UserIdCollectionMode
16+
import datadog.trace.api.telemetry.WafMetricCollector
1617
import datadog.trace.test.util.DDSpecification
1718

1819
import java.nio.file.Files
@@ -50,6 +51,7 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
5051
AppSecModuleConfigurer.Reconfiguration reconf = Stub()
5152
AppSecConfigServiceImpl appSecConfigService
5253
SavedListeners listeners
54+
protected static final ORIGINAL_METRIC_COLLECTOR = WafMetricCollector.get()
5355

5456
void cleanup() {
5557
appSecConfigService?.close()
@@ -712,6 +714,51 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
712714
AppSecSystem.active = true
713715
}
714716

717+
void 'InvalidRuleSetException is thrown when rules are not configured correctly' () {
718+
setup:
719+
// Mock WafMetricCollector
720+
WafMetricCollector wafMetricCollector = Mock(WafMetricCollector)
721+
WafMetricCollector.INSTANCE = wafMetricCollector
722+
723+
// Create a temporary file with invalid WAF configuration
724+
Path p = Files.createTempFile('appsec', '.json')
725+
p.toFile() << '''{
726+
"version": "2.2",
727+
"rules": [
728+
{
729+
"id": "invalid-rule",
730+
"name": "Invalid Rule",
731+
"tags": {
732+
"type": "invalid_type",
733+
"category": "invalid_category"
734+
},
735+
"conditions": [
736+
{
737+
"operator": "invalid_operator",
738+
"parameters": {
739+
"invalid_param": "invalid_value"
740+
}
741+
}
742+
],
743+
"type": "invalid_type",
744+
"data": []
745+
}
746+
]
747+
}'''
748+
749+
when:
750+
appSecConfigService.init()
751+
752+
then:
753+
1 * config.getAppSecRulesFile() >> (p as String)
754+
1 * wafMetricCollector.addWafConfigError(_ as Integer)
755+
thrown RuntimeException
756+
757+
cleanup:
758+
WafMetricCollector.INSTANCE = ORIGINAL_METRIC_COLLECTOR
759+
p.toFile().delete()
760+
}
761+
715762
private static AppSecFeatures autoUserInstrum(String mode) {
716763
return new AppSecFeatures().tap { features ->
717764
features.autoUserInstrum = new AppSecFeatures.AutoUserInstrum().tap { instrum ->

‎dd-java-agent/instrumentation/java-concurrent/src/test/groovy/QueueTimingForkedTest.groovy‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,20 @@ class QueueTimingForkedTest extends AgentTestRunner {
3131
})
3232

3333
then:
34-
verify(LinkedBlockingQueue.name)
34+
verify(LinkedBlockingQueue.name, 'TestRunnable')
3535

3636
when:
3737
runUnderTrace("parent", {
3838
fjp.submit(new TestRunnable()).get()
3939
})
4040

4141
then:
42+
// Starting from Java 24, ForkJoinPool will wrap a Runnable with the {@code java.util.concurrent.ForkJoinTask$AdaptedInterruptibleRunnable} class
43+
String expectedTaskClassName = Platform.isJavaVersionAtLeast(24) ? 'AdaptedInterruptibleRunnable' : 'TestRunnable'
44+
4245
// flaky before JDK21
4346
if (Platform.isJavaVersionAtLeast(21)) {
44-
verify("java.util.concurrent.ForkJoinPool\$WorkQueue")
47+
verify("java.util.concurrent.ForkJoinPool\$WorkQueue", expectedTaskClassName)
4548
}
4649

4750
cleanup:
@@ -50,15 +53,15 @@ class QueueTimingForkedTest extends AgentTestRunner {
5053
TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.clear()
5154
}
5255

53-
void verify(expectedQueueType) {
56+
void verify(expectedQueueType, expectedTaskClassName) {
5457
assert TEST_PROFILING_CONTEXT_INTEGRATION.isBalanced()
5558
assert !TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.isEmpty()
5659
int numAsserts = 0
5760
while (!TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.isEmpty()) {
5861
def timing = TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.takeFirst() as TestProfilingContextIntegration.TestQueueTiming
5962
if (!(timing.task as Class).simpleName.isEmpty()) {
6063
assert timing != null
61-
assert timing.task == TestRunnable
64+
assert timing.task.simpleName == expectedTaskClassName
6265
assert timing.scheduler != null
6366
assert timing.origin == Thread.currentThread()
6467
assert timing.queueLength >= 0

‎dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import datadog.trace.api.DisableTestTrace
22
import datadog.trace.api.civisibility.config.TestFQN
33
import datadog.trace.api.civisibility.config.TestIdentifier
4+
import datadog.trace.api.Platform
45
import datadog.trace.civisibility.CiVisibilityInstrumentationTest
56
import datadog.trace.instrumentation.karate.KarateUtils
67
import datadog.trace.instrumentation.karate.TestEventsHandlerHolder
@@ -13,12 +14,16 @@ import org.junit.platform.launcher.TestExecutionListener
1314
import org.junit.platform.launcher.core.LauncherConfig
1415
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder
1516
import org.junit.platform.launcher.core.LauncherFactory
17+
import spock.lang.IgnoreIf
1618

1719
import java.util.concurrent.ConcurrentHashMap
1820
import java.util.concurrent.CopyOnWriteArrayList
1921

2022
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass
2123

24+
@IgnoreIf(reason = "Karate instrumentation is creating an unclosed span of kind test_suite_end specifically for Java 24. Skip until we have a fix.", value = {
25+
Platform.isJavaVersionAtLeast(24)
26+
})
2227
@DisableTestTrace(reason = "avoid self-tracing")
2328
class KarateTest extends CiVisibilityInstrumentationTest {
2429

‎dd-java-agent/instrumentation/vertx-pg-client-4.0/src/test/groovy/VertxPostgresSqlClientForkedTest.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,18 @@ import io.vertx.pgclient.PgPool
1111
import io.vertx.sqlclient.*
1212
import spock.lang.AutoCleanup
1313
import spock.lang.Shared
14+
import spock.lang.IgnoreIf
1415

1516
import java.util.concurrent.CountDownLatch
1617
import java.util.concurrent.TimeUnit
1718

1819
import static datadog.trace.agent.test.utils.TraceUtils.basicSpan
1920
import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace
21+
import datadog.trace.api.Platform
2022

23+
@IgnoreIf(reason = "A change in Locale.ROOT that was introduced in JDK 22 is not fixed until vertx-pg-client v4.5.1: https://github.com/eclipse-vertx/vertx-sql-client/pull/1394", value = {
24+
Platform.isJavaVersionAtLeast(22)
25+
})
2126
class VertxPostgresSqlClientForkedTest extends AgentTestRunner {
2227
@AutoCleanup
2328
@Shared

‎dd-java-agent/instrumentation/wildfly-9/src/test/groovy/WildFlyForkedTest.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,18 @@ import datadog.trace.agent.test.utils.OkHttpUtils
44
import datadog.trace.api.DDSpanTypes
55
import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
66
import datadog.trace.bootstrap.instrumentation.api.Tags
7+
import datadog.trace.api.Platform
78
import okhttp3.HttpUrl
89
import okhttp3.Request
910
import org.jboss.shrinkwrap.api.ShrinkWrap
1011
import org.jboss.shrinkwrap.api.spec.WebArchive
12+
import spock.lang.IgnoreIf
1113
import test.JakartaTestServlet
1214
import test.TestServlet
1315

16+
@IgnoreIf(reason = "WildFly does not guarantee support for Java SE 24. The latest version 36 of WildFly recommends using the latest Java LTS: https://docs.wildfly.org/36/Getting_Started_Guide.html#requirements", value = {
17+
Platform.isJavaVersionAtLeast(22)
18+
})
1419
class WildFlyForkedTest extends WithHttpServer<EmbeddedWildfly> implements TestingGenericHttpNamingConventions.ServerV0 {
1520
@Override
1621
EmbeddedWildfly startServer(int port) {

0 commit comments

Comments
 (0)