Parallelize GitHub Action tests via a customTest task

.github/workflows/java-21-builds.yml

@@ -1,4 +1,4 @@
-name: Java 21 CI (MC 1.21+)
+name: Java 21 CI
 
 on:
     push:
@@ -8,9 +8,21 @@ on:
     pull_request:
 
 jobs:
-    build:
+    prepare:
+        name: Parallelize Tests
         if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
+        uses: ./.github/workflows/parallelize-tests.yml
+        with:
+            environments: 1.21,1.21.1,1.21.3,1.21.4,1.21.5,1.21.8,1.21.10
+            java_version: 21
+            parallel_jobs: 3
+
+    build:
+        name: ${{ matrix.display }}
+        needs: prepare
         runs-on: ubuntu-latest
+        strategy:
+            matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
         steps:
             - uses: actions/checkout@v5
               with:
@@ -26,10 +38,10 @@ jobs:
             - name: Grant execute permission for gradlew
               run: chmod +x gradlew
             - name: Build Skript and run test scripts
-              run: ./gradlew clean skriptTestJava21
+              run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21
             - name: Upload Nightly Build
               uses: actions/upload-artifact@v4
-              if: success()
+              if: success() && matrix.id == 1
               with:
                   name: skript-nightly
                   path: build/libs/*

.github/workflows/junit-21-builds.yml

@@ -1,4 +1,4 @@
-name: JUnit (MC 1.21+)
+name: JUnit 21
 
 on:
     push:
@@ -8,9 +8,21 @@ on:
     pull_request:
 
 jobs:
-    build:
+    prepare:
+        name: Parallelize Tests
         if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
+        uses: ./.github/workflows/parallelize-tests.yml
+        with:
+            environments: 1.21,1.21.1,1.21.3,1.21.4,1.21.5,1.21.8,1.21.10
+            java_version: 21
+            parallel_jobs: 3
+
+    build:
+        name: ${{ matrix.display }}
+        needs: prepare
         runs-on: ubuntu-latest
+        strategy:
+            matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
         steps:
             - uses: actions/checkout@v5
               with:
@@ -25,5 +37,11 @@ jobs:
                   cache: gradle
             - name: Grant execute permission for gradlew
               run: chmod +x gradlew
-            - name: Build Skript and run JUnit
-              run: ./gradlew clean JUnitJava21
+            - name: Build Skript and run test scripts
+              run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21 -Pjunit=true
+            - name: Upload Nightly Build
+              uses: actions/upload-artifact@v4
+              if: success() && matrix.id == 1
+              with:
+                  name: skript-nightly
+                  path: build/libs/*

.github/workflows/parallelize-tests.yml

@@ -0,0 +1,85 @@
+name: Parallelize Tests
+
+on:
+    workflow_call:
+        inputs:
+            environments:
+                required: true
+                type: string
+            parallel_jobs:
+                required: false
+                type: number
+                default: 2
+            java_version:
+                required: true
+                type: number
+        outputs:
+            matrix:
+                description: "Generated test matrix"
+                value: ${{ jobs.prepare.outputs.matrix }}
+
+jobs:
+    prepare:
+        name: ""
+        runs-on: ubuntu-latest
+        outputs:
+            matrix: ${{ steps.set-matrix.outputs.matrix }}
+        steps:
+            - name: Generate matrix
+              id: set-matrix
+              run: |
+                  # Use environment variables
+                  N=${{ inputs.parallel_jobs }}
+                  JAVA_VERSION=${{ inputs.java_version }}
+
+                  # Convert to array and transform
+                  IFS=',' read -ra ENVS <<< "${{ inputs.environments }}"
+                  TRANSFORMED=()
+                  for env in "${ENVS[@]}"; do
+                      TRANSFORMED+=("java${JAVA_VERSION}/paper-${env}")
+                  done
+
+                  TOTAL=${#TRANSFORMED[@]}
+
+                  # avoid creating more jobs than needed
+                  JOBS=$((TOTAL < N ? TOTAL : N))
+                  # environments per job
+                  BASE=$((TOTAL / JOBS))
+                  EXTRA=$((TOTAL % JOBS))
+
+                  # Build matrix JSON
+                  # Format is:
+                  # [{"id":1,"envs":"java21/paper-1.20.6,java21/paper-1.21.3","display":"1.20.6, 1.21.3"},...]
+                  # Where "envs" is the actual environment strings and "display" is for easier identification
+                  MATRIX="["
+                  INDEX=0
+                  for ((i=0; i<JOBS; i++)); do
+                      # Determine count for this job (BASE + 1 if i < EXTRA)
+                      COUNT=$BASE
+                      if [ $i -lt $EXTRA ]; then
+                          COUNT=$((COUNT + 1))
+                      fi
+
+                      # Build envs and display strings
+                      GROUP=""
+                      DISPLAY_GROUP=""
+                      for ((j=0; j<COUNT; j++)); do
+                          if [ -n "$GROUP" ]; then
+                              GROUP="$GROUP,"
+                              DISPLAY_GROUP="$DISPLAY_GROUP, "
+                          fi
+                          GROUP="$GROUP${TRANSFORMED[$INDEX]}"
+                          DISPLAY_GROUP="$DISPLAY_GROUP${ENVS[$INDEX]}"
+                          INDEX=$((INDEX + 1))
+                      done
+
+                      # add to matrix with separating comma if needed
+                      if [ $i -gt 0 ]; then
+                          MATRIX="$MATRIX,"
+                      fi
+                      MATRIX="$MATRIX{\"id\":$((i+1)),\"envs\":\"$GROUP\",\"display\":\"$DISPLAY_GROUP\"}"
+                  done
+                  MATRIX="$MATRIX]"
+
+                  echo "matrix={\"include\":$MATRIX}" >> $GITHUB_OUTPUT
+                  echo "Generated matrix: {\"include\":$MATRIX}"

build.gradle

@@ -286,6 +286,31 @@ tasks.register('JUnit') {
 	dependsOn JUnitJava21
 }
 
+// custom test task
+// usage: gradle customTest -PtestEnvs=<environment files, comma separated list.> -PtestEnvJavaVersion=<java version> -Ptimeout=<timeout in ms> -Pjunit=[true|false]
+// defaults: testEnvJavaVersion=latestJava, timeout=0, junit=false
+// example: gradle customTest -PtestEnvs="java21/paper-1.21.4,java21/paper-1.21.8" -PtestEnvJavaVersion=21 -Ptimeout=600000 -Pjunit=true
+
+// get environments
+String propEnvs = project.hasProperty('testEnvs') ? project.property('testEnvs') as String : project.testEnv
+String[] envList = propEnvs != null ? propEnvs.split(',') : [env]
+StringBuilder customEnvironmentsBuilder = new StringBuilder()
+for (int i = 0; i < envList.length; i++) {
+	customEnvironmentsBuilder.append(environments).append(envList[i]).append('.json')
+	if (i < envList.length - 1) {
+		customEnvironmentsBuilder.append(',') // use ',' as separator for multiple environments
+	}
+}
+String customEnvironments = customEnvironmentsBuilder.toString()
+
+long customTimeout = project.hasProperty('timeout') ? Long.parseLong(project.property('timeout') as String) : 0
+List<Modifiers> customModifiers = new ArrayList<>()
+if (project.hasProperty('junit') && (project.property('junit') as String).toLowerCase() == 'true') {
+	customModifiers.add(Modifiers.JUNIT)
+}
+createTestTask('customTest', 'Runs tests based on provided parameters.', customEnvironments, envJava, customTimeout, customModifiers.toArray(new Modifiers[0]) as Modifiers[])
+// end custom test task
+
 // Build flavor configurations
 tasks.register('githubResources', ProcessResources) {
 	from 'src/main/resources', {

src/main/java/ch/njol/skript/Skript.java

@@ -1307,12 +1307,15 @@ public void onDisable() {
 			return;
 		disabled = true;
 
+		info("beforeDisable");
 		if (!partDisabled) {
 			beforeDisable();
 		}
 
+		info("cancelTasks");
 		Bukkit.getScheduler().cancelTasks(this);
 
+		info("closing closables");
 		for (Closeable c : closeOnDisable) {
 			try {
 				c.close();
@@ -1321,6 +1324,7 @@ public void onDisable() {
 			}
 		}
 
+		info("done with disable");
 		this.experimentRegistry = null;
 	}
 

src/main/java/ch/njol/skript/test/platform/PlatformMain.java

@@ -1,28 +1,17 @@
 package ch.njol.skript.test.platform;
 
 import ch.njol.skript.test.utils.TestResults;
-import ch.njol.util.NonNullPair;
 import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonSyntaxException;
 import org.apache.commons.lang.StringUtils;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -36,13 +25,16 @@ public static void main(String... args) throws IOException, InterruptedException
 		Gson gson = new GsonBuilder().setPrettyPrinting().create();
 
 		Path runnerRoot = Paths.get(args[0]);
-		assert runnerRoot != null;
 		Path testsRoot = Paths.get(args[1]).toAbsolutePath();
-		assert testsRoot != null;
 		Path dataRoot = Paths.get(args[2]);
-		assert dataRoot != null;
-		Path envsRoot = Paths.get(args[3]);
-		assert envsRoot != null;
+		// allow multiple environments separated by commas
+		List<Path> envPaths = new ArrayList<>();
+		String envsArg = args[3];
+		envsArg = envsArg.trim();
+		String[] envPathStrings = envsArg.split(",");
+		for (String envPath : envPathStrings) {
+			envPaths.add(Paths.get(envPath.trim()));
+		}
 		boolean devMode = "true".equals(args[4]);
 		boolean genDocs = "true".equals(args[5]);
 		boolean jUnit = "true".equals(args[6]);
@@ -56,27 +48,29 @@ public static void main(String... args) throws IOException, InterruptedException
 			jvmArgs.add("-Xmx5G");
 
 		// Load environments
-		List<Environment> envs;
-		if (Files.isDirectory(envsRoot)) {
-			envs = Files.walk(envsRoot).filter(path -> !Files.isDirectory(path))
+		List<Environment> envs = new ArrayList<>();
+		for (Path envPath : envPaths) {
+			if (Files.isDirectory(envPath)) {
+				envs.addAll(Files.walk(envPath).filter(path -> !Files.isDirectory(path))
 					.map(path -> {
 						try {
-							return gson.fromJson(new String(Files.readAllBytes(path), StandardCharsets.UTF_8), Environment.class);
+							return gson.fromJson(Files.readString(path), Environment.class);
 						} catch (JsonSyntaxException | IOException e) {
 							throw new RuntimeException(e);
 						}
-					}).collect(Collectors.toList());
-		} else {
-			envs = Collections.singletonList(gson.fromJson(new String(
-					Files.readAllBytes(envsRoot),StandardCharsets.UTF_8), Environment.class));
+					}).toList());
+			} else {
+				envs.add(gson.fromJson(Files.readString(envPath), Environment.class));
+			}
 		}
-		System.out.println("Test environments: " + String.join(", ",
-				envs.stream().map(Environment::getName).collect(Collectors.toList())));
+		System.out.println("Test environments: "
+				+ envs.stream().map(Environment::getName).collect(Collectors.joining(", ")));
 
 		Set<String> allTests = new HashSet<>();
-		Map<String, List<NonNullPair<Environment, String>>> failures = new HashMap<>();
+		Map<String, List<TestError>> failures = new HashMap<>();
 
 		boolean docsFailed = false;
+		Map<Environment, TestResults> collectedResults = Collections.synchronizedMap(new HashMap<>());
 		// Run tests and collect the results
 		envs.sort(Comparator.comparing(Environment::getName));
 		for (Environment env : envs) {
@@ -93,16 +87,21 @@ public static void main(String... args) throws IOException, InterruptedException
 				System.exit(3);
 				return;
 			}
-
-			// Collect results
-			docsFailed = results.docsFailed();
+			collectedResults.put(env, results);
+		}
+
+		// Process collected results
+		for (var entry : collectedResults.entrySet()) {
+			TestResults results = entry.getValue();
+			Environment env = entry.getKey();
+			docsFailed |= results.docsFailed();
 			allTests.addAll(results.getSucceeded());
 			allTests.addAll(results.getFailed().keySet());
 			for (Map.Entry<String, String> fail : results.getFailed().entrySet()) {
 				String error = fail.getValue();
 				assert error != null;
 				failures.computeIfAbsent(fail.getKey(), (k) -> new ArrayList<>())
-						.add(new NonNullPair<>(env, error));
+					.add(new TestError(env, error));
 			}
 		}
 
@@ -119,33 +118,33 @@ public static void main(String... args) throws IOException, InterruptedException
 		}
 
 		// Sort results in alphabetical order
-		List<String> succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).collect(Collectors.toList());
-		Collections.sort(succeeded);
+		List<String> succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).sorted().collect(Collectors.toList());
 		List<String> failNames = new ArrayList<>(failures.keySet());
 		Collections.sort(failNames);
 
 		// All succeeded tests in a single line
 		StringBuilder output = new StringBuilder(String.format("%s Results %s%n", StringUtils.repeat("-", 25), StringUtils.repeat("-", 25)));
-		output.append("\nTested environments: " + String.join(", ",
-				envs.stream().map(Environment::getName).collect(Collectors.toList())));
-		output.append("\nSucceeded:\n  " + String.join((jUnit ? "\n  " : ", "), succeeded));
+		output.append("\nTested environments: ").append(envs.stream().map(Environment::getName).collect(Collectors.joining(", ")));
+		output.append("\nSucceeded:\n  ").append(String.join((jUnit ? "\n  " : ", "), succeeded));
 
 		if (!failNames.isEmpty()) { // More space for failed tests, they're important
 			output.append("\nFailed:");
 			for (String failed : failNames) {
-				List<NonNullPair<Environment, String>> errors = failures.get(failed);
-				output.append("\n  " + failed + " (on " + errors.size() + " environment" + (errors.size() == 1 ? "" : "s") + ")");
-				for (NonNullPair<Environment, String> error : errors) {
-					output.append("\n    " + error.getSecond() + " (on " + error.getFirst().getName() + ")");
+				List<TestError> errors = failures.get(failed);
+				output.append("\n  ").append(failed).append(" (on ").append(errors.size()).append(" environment").append(errors.size() == 1 ? "" : "s").append(")");
+				for (TestError error : errors) {
+					output.append("\n    ").append(error.message()).append(" (on ").append(error.environment().getName()).append(")");
 				}
 			}
 			output.append(String.format("%n%n%s", StringUtils.repeat("-", 60)));
-			System.err.print(output.toString());
+			System.err.print(output);
 			System.exit(failNames.size()); // Error code to indicate how many tests failed.
 			return;
 		}
 		output.append(String.format("%n%n%s", StringUtils.repeat("-", 60)));
-		System.out.print(output.toString());
+		System.out.print(output);
 	}
 
+	private record TestError(Environment environment, String message) { }
+
 }