ci: run aggtests in parallel using xargs

swanandx · swanandx · commit d6d583ec10ed · 2025-11-25T11:44:02.000Z
Our ci has parallel compilation setup, so multiple pipelines can be
compiled at once. We use xargs -P to run runtime_aggtests in parallel
so we can take advantage of it and speed up our ci.

Signed-off-by: Swanand Mulay &lt;73115739+swanandx@users.noreply.github.com&gt;
diff --git a/.github/workflows/test-integration-runtime.yml b/.github/workflows/test-integration-runtime.yml
@@ -51,6 +51,7 @@ jobs:
         run: uv run --locked ./tests/runtime_aggtest/run.sh
         working-directory: python
         env:
+          RUNTIME_AGGTEST_JOBS: ${{ vars.RUNTIME_AGGTEST_JOBS }}
           PYTHONPATH: ${{ github.workspace }}/python
           FELDERA_TLS_INSECURE: true
 
diff --git a/python/tests/runtime_aggtest/run.sh b/python/tests/runtime_aggtest/run.sh
@@ -2,19 +2,29 @@
 
 set -ex
 
-function runtest() {
+TESTS=(
+    "aggregate_tests/main.py"
+    "aggregate_tests2/main.py"
+    "aggregate_tests3/main.py"
+    "aggregate_tests4/main.py"
+    "aggregate_tests5/main.py"
+    "aggregate_tests6/main.py"
+    "arithmetic_tests/main.py"
+    "asof_tests/main.py"
+    "complex_type_tests/main.py"
+    "illarg_tests/main.py"
+    "negative_tests/main.py"
+    "orderby_tests/main.py"
+    "variant_tests/main.py"
+)
+
+function run_one() {
     uv run --locked $PYTHONPATH/tests/runtime_aggtest/$1
 }
 
-runtest aggregate_tests/main.py
-runtest aggregate_tests2/main.py
-runtest aggregate_tests3/main.py
-runtest aggregate_tests4/main.py
-runtest aggregate_tests5/main.py
-runtest aggregate_tests6/main.py
-runtest arithmetic_tests/main.py
-runtest complex_type_tests/main.py
-runtest orderby_tests/main.py
-runtest variant_tests/main.py
-runtest illarg_tests/main.py
-runtest negative_tests/main.py
+if [ "${RUNTIME_AGGTEST_JOBS:-1}" -le 1 ]; then
+  for t in "${TESTS[@]}"; do run_one "$t"; done
+else
+  echo "Running tests in parallel: ${RUNTIME_AGGTEST_JOBS} jobs"
+  printf '%s\n' "${TESTS[@]}" | xargs -P "${RUNTIME_AGGTEST_JOBS}" -I{} bash -e -c 'echo "Running: {}"; uv run --locked "$PYTHONPATH/tests/runtime_aggtest/{}"'
+fi
