Skip to content

Commit 24a2435

Browse files
sovdeethsovdeeth
authored
Parallelize GitHub Action tests via a customTest task (#8317)
* paralleize github actions via customTest task * test test collection failures * remove debug messages * remove junit nightly
1 parent ba02475 commit 24a2435

6 files changed

Lines changed: 201 additions & 51 deletions

File tree

‎.github/workflows/java-21-builds.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Java 21 CI (MC 1.21+)
1+
name: Java 21 CI
22

33
on:
44
push:
@@ -8,9 +8,21 @@ on:
88
pull_request:
99

1010
jobs:
11-
build:
11+
prepare:
12+
name: Parallelize Tests
1213
if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
14+
uses: ./.github/workflows/parallelize-tests.yml
15+
with:
16+
environments: 1.21,1.21.1,1.21.3,1.21.4,1.21.5,1.21.8,1.21.10
17+
java_version: 21
18+
parallel_jobs: 3
19+
20+
build:
21+
name: ${{ matrix.display }}
22+
needs: prepare
1323
runs-on: ubuntu-latest
24+
strategy:
25+
matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
1426
steps:
1527
- uses: actions/checkout@v5
1628
with:
@@ -26,10 +38,10 @@ jobs:
2638
- name: Grant execute permission for gradlew
2739
run: chmod +x gradlew
2840
- name: Build Skript and run test scripts
29-
run: ./gradlew clean skriptTestJava21
41+
run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21
3042
- name: Upload Nightly Build
3143
uses: actions/upload-artifact@v4
32-
if: success()
44+
if: success() && matrix.id == 1
3345
with:
3446
name: skript-nightly
3547
path: build/libs/*

‎.github/workflows/junit-21-builds.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: JUnit (MC 1.21+)
1+
name: JUnit 21
22

33
on:
44
push:
@@ -8,9 +8,21 @@ on:
88
pull_request:
99

1010
jobs:
11-
build:
11+
prepare:
12+
name: Parallelize Tests
1213
if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
14+
uses: ./.github/workflows/parallelize-tests.yml
15+
with:
16+
environments: 1.21,1.21.1,1.21.3,1.21.4,1.21.5,1.21.8,1.21.10
17+
java_version: 21
18+
parallel_jobs: 3
19+
20+
build:
21+
name: ${{ matrix.display }}
22+
needs: prepare
1323
runs-on: ubuntu-latest
24+
strategy:
25+
matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
1426
steps:
1527
- uses: actions/checkout@v5
1628
with:
@@ -25,5 +37,5 @@ jobs:
2537
cache: gradle
2638
- name: Grant execute permission for gradlew
2739
run: chmod +x gradlew
28-
- name: Build Skript and run JUnit
29-
run: ./gradlew clean JUnitJava21
40+
- name: Build Skript and run test scripts
41+
run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21 -Pjunit=true

‎.github/workflows/parallelize-tests.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Parallelize Tests
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
environments:
7+
required: true
8+
type: string
9+
parallel_jobs:
10+
required: false
11+
type: number
12+
default: 2
13+
java_version:
14+
required: true
15+
type: number
16+
outputs:
17+
matrix:
18+
description: "Generated test matrix"
19+
value: ${{ jobs.prepare.outputs.matrix }}
20+
21+
jobs:
22+
prepare:
23+
name: ""
24+
runs-on: ubuntu-latest
25+
outputs:
26+
matrix: ${{ steps.set-matrix.outputs.matrix }}
27+
steps:
28+
- name: Generate matrix
29+
id: set-matrix
30+
run: |
31+
# Use environment variables
32+
N=${{ inputs.parallel_jobs }}
33+
JAVA_VERSION=${{ inputs.java_version }}
34+
35+
# Convert to array and transform
36+
IFS=',' read -ra ENVS <<< "${{ inputs.environments }}"
37+
TRANSFORMED=()
38+
for env in "${ENVS[@]}"; do
39+
TRANSFORMED+=("java${JAVA_VERSION}/paper-${env}")
40+
done
41+
42+
TOTAL=${#TRANSFORMED[@]}
43+
44+
# avoid creating more jobs than needed
45+
JOBS=$((TOTAL < N ? TOTAL : N))
46+
# environments per job
47+
BASE=$((TOTAL / JOBS))
48+
EXTRA=$((TOTAL % JOBS))
49+
50+
# Build matrix JSON
51+
# Format is:
52+
# [{"id":1,"envs":"java21/paper-1.20.6,java21/paper-1.21.3","display":"1.20.6, 1.21.3"},...]
53+
# Where "envs" is the actual environment strings and "display" is for easier identification
54+
MATRIX="["
55+
INDEX=0
56+
for ((i=0; i<JOBS; i++)); do
57+
# Determine count for this job (BASE + 1 if i < EXTRA)
58+
COUNT=$BASE
59+
if [ $i -lt $EXTRA ]; then
60+
COUNT=$((COUNT + 1))
61+
fi
62+
63+
# Build envs and display strings
64+
GROUP=""
65+
DISPLAY_GROUP=""
66+
for ((j=0; j<COUNT; j++)); do
67+
if [ -n "$GROUP" ]; then
68+
GROUP="$GROUP,"
69+
DISPLAY_GROUP="$DISPLAY_GROUP, "
70+
fi
71+
GROUP="$GROUP${TRANSFORMED[$INDEX]}"
72+
DISPLAY_GROUP="$DISPLAY_GROUP${ENVS[$INDEX]}"
73+
INDEX=$((INDEX + 1))
74+
done
75+
76+
# add to matrix with separating comma if needed
77+
if [ $i -gt 0 ]; then
78+
MATRIX="$MATRIX,"
79+
fi
80+
MATRIX="$MATRIX{\"id\":$((i+1)),\"envs\":\"$GROUP\",\"display\":\"$DISPLAY_GROUP\"}"
81+
done
82+
MATRIX="$MATRIX]"
83+
84+
echo "matrix={\"include\":$MATRIX}" >> $GITHUB_OUTPUT
85+
echo "Generated matrix: {\"include\":$MATRIX}"

‎build.gradle‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,31 @@ tasks.register('JUnit') {
286286
dependsOn JUnitJava21
287287
}
288288

289+
// custom test task
290+
// usage: gradle customTest -PtestEnvs=<environment files, comma separated list.> -PtestEnvJavaVersion=<java version> -Ptimeout=<timeout in ms> -Pjunit=[true|false]
291+
// defaults: testEnvJavaVersion=latestJava, timeout=0, junit=false
292+
// example: gradle customTest -PtestEnvs="java21/paper-1.21.4,java21/paper-1.21.8" -PtestEnvJavaVersion=21 -Ptimeout=600000 -Pjunit=true
293+
294+
// get environments
295+
String propEnvs = project.hasProperty('testEnvs') ? project.property('testEnvs') as String : project.testEnv
296+
String[] envList = propEnvs != null ? propEnvs.split(',') : [env]
297+
StringBuilder customEnvironmentsBuilder = new StringBuilder()
298+
for (int i = 0; i < envList.length; i++) {
299+
customEnvironmentsBuilder.append(environments).append(envList[i]).append('.json')
300+
if (i < envList.length - 1) {
301+
customEnvironmentsBuilder.append(',') // use ',' as separator for multiple environments
302+
}
303+
}
304+
String customEnvironments = customEnvironmentsBuilder.toString()
305+
306+
long customTimeout = project.hasProperty('timeout') ? Long.parseLong(project.property('timeout') as String) : 0
307+
List<Modifiers> customModifiers = new ArrayList<>()
308+
if (project.hasProperty('junit') && (project.property('junit') as String).toLowerCase() == 'true') {
309+
customModifiers.add(Modifiers.JUNIT)
310+
}
311+
createTestTask('customTest', 'Runs tests based on provided parameters.', customEnvironments, envJava, customTimeout, customModifiers.toArray(new Modifiers[0]) as Modifiers[])
312+
// end custom test task
313+
289314
// Build flavor configurations
290315
tasks.register('githubResources', ProcessResources) {
291316
from 'src/main/resources', {

‎src/main/java/ch/njol/skript/test/platform/PlatformMain.java‎

Lines changed: 42 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,17 @@
11
package ch.njol.skript.test.platform;
22

33
import ch.njol.skript.test.utils.TestResults;
4-
import ch.njol.util.NonNullPair;
54
import com.google.common.collect.Sets;
65
import com.google.gson.Gson;
76
import com.google.gson.GsonBuilder;
87
import com.google.gson.JsonSyntaxException;
98
import org.apache.commons.lang.StringUtils;
109

1110
import java.io.IOException;
12-
import java.nio.charset.StandardCharsets;
1311
import java.nio.file.Files;
1412
import java.nio.file.Path;
1513
import java.nio.file.Paths;
16-
import java.util.ArrayList;
17-
import java.util.Arrays;
18-
import java.util.Collections;
19-
import java.util.Comparator;
20-
import java.util.HashMap;
21-
import java.util.HashSet;
22-
import java.util.List;
23-
import java.util.Locale;
24-
import java.util.Map;
25-
import java.util.Set;
14+
import java.util.*;
2615
import java.util.stream.Collectors;
2716

2817
/**
@@ -36,13 +25,16 @@ public static void main(String... args) throws IOException, InterruptedException
3625
Gson gson = new GsonBuilder().setPrettyPrinting().create();
3726

3827
Path runnerRoot = Paths.get(args[0]);
39-
assert runnerRoot != null;
4028
Path testsRoot = Paths.get(args[1]).toAbsolutePath();
41-
assert testsRoot != null;
4229
Path dataRoot = Paths.get(args[2]);
43-
assert dataRoot != null;
44-
Path envsRoot = Paths.get(args[3]);
45-
assert envsRoot != null;
30+
// allow multiple environments separated by commas
31+
List<Path> envPaths = new ArrayList<>();
32+
String envsArg = args[3];
33+
envsArg = envsArg.trim();
34+
String[] envPathStrings = envsArg.split(",");
35+
for (String envPath : envPathStrings) {
36+
envPaths.add(Paths.get(envPath.trim()));
37+
}
4638
boolean devMode = "true".equals(args[4]);
4739
boolean genDocs = "true".equals(args[5]);
4840
boolean jUnit = "true".equals(args[6]);
@@ -56,27 +48,29 @@ public static void main(String... args) throws IOException, InterruptedException
5648
jvmArgs.add("-Xmx5G");
5749

5850
// Load environments
59-
List<Environment> envs;
60-
if (Files.isDirectory(envsRoot)) {
61-
envs = Files.walk(envsRoot).filter(path -> !Files.isDirectory(path))
51+
List<Environment> envs = new ArrayList<>();
52+
for (Path envPath : envPaths) {
53+
if (Files.isDirectory(envPath)) {
54+
envs.addAll(Files.walk(envPath).filter(path -> !Files.isDirectory(path))
6255
.map(path -> {
6356
try {
64-
return gson.fromJson(new String(Files.readAllBytes(path), StandardCharsets.UTF_8), Environment.class);
57+
return gson.fromJson(Files.readString(path), Environment.class);
6558
} catch (JsonSyntaxException | IOException e) {
6659
throw new RuntimeException(e);
6760
}
68-
}).collect(Collectors.toList());
69-
} else {
70-
envs = Collections.singletonList(gson.fromJson(new String(
71-
Files.readAllBytes(envsRoot),StandardCharsets.UTF_8), Environment.class));
61+
}).toList());
62+
} else {
63+
envs.add(gson.fromJson(Files.readString(envPath), Environment.class));
64+
}
7265
}
73-
System.out.println("Test environments: " + String.join(", ",
74-
envs.stream().map(Environment::getName).collect(Collectors.toList())));
66+
System.out.println("Test environments: "
67+
+ envs.stream().map(Environment::getName).collect(Collectors.joining(", ")));
7568

7669
Set<String> allTests = new HashSet<>();
77-
Map<String, List<NonNullPair<Environment, String>>> failures = new HashMap<>();
70+
Map<String, List<TestError>> failures = new HashMap<>();
7871

7972
boolean docsFailed = false;
73+
Map<Environment, TestResults> collectedResults = Collections.synchronizedMap(new HashMap<>());
8074
// Run tests and collect the results
8175
envs.sort(Comparator.comparing(Environment::getName));
8276
for (Environment env : envs) {
@@ -93,16 +87,21 @@ public static void main(String... args) throws IOException, InterruptedException
9387
System.exit(3);
9488
return;
9589
}
96-
97-
// Collect results
98-
docsFailed = results.docsFailed();
90+
collectedResults.put(env, results);
91+
}
92+
93+
// Process collected results
94+
for (var entry : collectedResults.entrySet()) {
95+
TestResults results = entry.getValue();
96+
Environment env = entry.getKey();
97+
docsFailed |= results.docsFailed();
9998
allTests.addAll(results.getSucceeded());
10099
allTests.addAll(results.getFailed().keySet());
101100
for (Map.Entry<String, String> fail : results.getFailed().entrySet()) {
102101
String error = fail.getValue();
103102
assert error != null;
104103
failures.computeIfAbsent(fail.getKey(), (k) -> new ArrayList<>())
105-
.add(new NonNullPair<>(env, error));
104+
.add(new TestError(env, error));
106105
}
107106
}
108107

@@ -119,33 +118,33 @@ public static void main(String... args) throws IOException, InterruptedException
119118
}
120119

121120
// Sort results in alphabetical order
122-
List<String> succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).collect(Collectors.toList());
123-
Collections.sort(succeeded);
121+
List<String> succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).sorted().collect(Collectors.toList());
124122
List<String> failNames = new ArrayList<>(failures.keySet());
125123
Collections.sort(failNames);
126124

127125
// All succeeded tests in a single line
128126
StringBuilder output = new StringBuilder(String.format("%s Results %s%n", StringUtils.repeat("-", 25), StringUtils.repeat("-", 25)));
129-
output.append("\nTested environments: " + String.join(", ",
130-
envs.stream().map(Environment::getName).collect(Collectors.toList())));
131-
output.append("\nSucceeded:\n " + String.join((jUnit ? "\n " : ", "), succeeded));
127+
output.append("\nTested environments: ").append(envs.stream().map(Environment::getName).collect(Collectors.joining(", ")));
128+
output.append("\nSucceeded:\n ").append(String.join((jUnit ? "\n " : ", "), succeeded));
132129

133130
if (!failNames.isEmpty()) { // More space for failed tests, they're important
134131
output.append("\nFailed:");
135132
for (String failed : failNames) {
136-
List<NonNullPair<Environment, String>> errors = failures.get(failed);
137-
output.append("\n " + failed + " (on " + errors.size() + " environment" + (errors.size() == 1 ? "" : "s") + ")");
138-
for (NonNullPair<Environment, String> error : errors) {
139-
output.append("\n " + error.getSecond() + " (on " + error.getFirst().getName() + ")");
133+
List<TestError> errors = failures.get(failed);
134+
output.append("\n ").append(failed).append(" (on ").append(errors.size()).append(" environment").append(errors.size() == 1 ? "" : "s").append(")");
135+
for (TestError error : errors) {
136+
output.append("\n ").append(error.message()).append(" (on ").append(error.environment().getName()).append(")");
140137
}
141138
}
142139
output.append(String.format("%n%n%s", StringUtils.repeat("-", 60)));
143-
System.err.print(output.toString());
140+
System.err.print(output);
144141
System.exit(failNames.size()); // Error code to indicate how many tests failed.
145142
return;
146143
}
147144
output.append(String.format("%n%n%s", StringUtils.repeat("-", 60)));
148-
System.out.print(output.toString());
145+
System.out.print(output);
149146
}
150147

148+
private record TestError(Environment environment, String message) { }
149+
151150
}

‎src/test/skript/environments/java21/paper-1.21.1.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "paper-1.21.1",
3+
"resources": [
4+
{"source": "server.properties.generic", "target": "server.properties"}
5+
],
6+
"paperDownloads": [
7+
{
8+
"version": "1.21.1",
9+
"target": "paperclip.jar"
10+
}
11+
],
12+
"skriptTarget": "plugins/Skript.jar",
13+
"commandLine": [
14+
"-Dcom.mojang.eula.agree=true",
15+
"-jar", "paperclip.jar", "--nogui"
16+
]
17+
}

0 commit comments

Comments
 (0)