Skip to content

Commit f2ce4a0

Browse files
partheaparthea
committed
tests: parallel batching for packages when running unit tests
1 parent 0f4bfed commit f2ce4a0

2 files changed

Lines changed: 124 additions & 10 deletions

File tree

‎.github/workflows/unittest.yml‎

Lines changed: 101 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,71 @@ name: unittest
1212
permissions:
1313
contents: read
1414

15+
# Configurable global environment variables for batching
16+
env:
17+
BATCH_SIZE: 10
18+
TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff
19+
1520
jobs:
21+
# Dynamic package discovery job to calculate required matrix size automatically
22+
discover-packages:
23+
runs-on: ubuntu-latest
24+
outputs:
25+
batch-indices: ${{ steps.set-matrix.outputs.indices }}
26+
steps:
27+
- name: Checkout
28+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
29+
with:
30+
persist-credentials: false
31+
- name: Generate Batch Indices
32+
id: set-matrix
33+
run: |
34+
# Testing all monorepo packages sequentially on a single runner node is
35+
# too slow and creates a severe CI bottleneck as the repository expands.
36+
#
37+
# To scale efficiently, we chunk the workload into parallel batch slices.
38+
# Instead of using a fixed, hardcoded matrix array (which risks silently
39+
# skipping newly added packages if the repository outgrows the array size),
40+
# this step dynamically audits the 'packages/' directory at runtime.
41+
#
42+
# It calculates exactly how many concurrent runners are required based on
43+
# the current repository size and the configured BATCH_SIZE variable,
44+
# ensuring 100% test coverage with zero manual YAML maintenance.
45+
46+
# 1. Count the number of total directories matching the 'packages/*' pattern.
47+
# Redirect stderr to /dev/null so empty repos do not print unneeded errors.
48+
TOTAL_PACKAGES=$(ls -d packages/*/ 2>/dev/null | wc -l | tr -d ' ')
49+
50+
# 2. Safety fallback: If no packages are detected, assign a single slice index [0]
51+
# so subsequent matrix-dependent jobs do not break or fail validation on an empty matrix.
52+
if [ "$TOTAL_PACKAGES" -eq 0 ]; then
53+
echo "indices=[0]" >> "$GITHUB_OUTPUT"
54+
exit 0
55+
fi
56+
57+
# 3. Calculate the number of batches required using ceiling division: ceil(TOTAL_PACKAGES / BATCH_SIZE).
58+
# The formula ((A + B - 1) / B) ensures integer division rounds up if there's any remaining package leftover.
59+
# Example: 251 packages with a batch size of 10 gives ((251 + 10 - 1) / 10) = 260 / 10 = 26 batches.
60+
NUM_BATCHES=$(( (TOTAL_PACKAGES + ${{ env.BATCH_SIZE }} - 1) / ${{ env.BATCH_SIZE }} ))
61+
62+
# 4. Generate a zero-indexed sequence from 0 to (NUM_BATCHES - 1).
63+
# Use jq to securely parse the raw numbers and compile them into a compacted JSON array string.
64+
# Example output format: [0,1,2,3,...,25]
65+
INDICES=$(seq 0 $((NUM_BATCHES - 1)) | jq -R . | jq -s -c .)
66+
67+
# 5. Output the finished JSON string to the GitHub environment outputs pipeline.
68+
# This will safely feed directly into the execution matrix downstream.
69+
echo "indices=${INDICES}" >> "$GITHUB_OUTPUT"
70+
1671
unit:
72+
name: "unit-run (${{ matrix.python }}, Batch ${{ matrix.batch-index }})"
1773
runs-on: ubuntu-22.04
74+
needs: discover-packages
1875
strategy:
19-
fail-fast: true
2076
matrix:
21-
python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
77+
python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14", "3.15"]
78+
# Dynamically scales to fit every package perfectly without hardcoding array indices
79+
batch-index: ${{ fromJson(needs.discover-packages.outputs.batch-indices) }}
2280
steps:
2381
- name: Checkout
2482
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -39,23 +97,41 @@ jobs:
3997
- name: Run unit tests
4098
env:
4199
COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
42-
BUILD_TYPE: presubmit
100+
# Dynamically set BUILD_TYPE to an empty string to skip the diff calculation if TEST_ALL_PACKAGES is true
101+
BUILD_TYPE: ${{ env.TEST_ALL_PACKAGES == 'true' && '' || 'presubmit' }}
43102
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
44103
TEST_TYPE: unit
45104
PY_VERSION: ${{ matrix.python }}
46105
run: |
47-
ci/run_conditional_tests.sh
106+
# Gather all packages in alphabetical order
107+
ALL_PACKAGES=($(ls -d packages/*/ | sort))
108+
TOTAL_PACKAGES=${#ALL_PACKAGES[@]}
109+
110+
# Determine this runner's slice window
111+
START_INDEX=$(( ${{ matrix.batch-index }} * ${{ env.BATCH_SIZE }} ))
112+
113+
if [ $START_INDEX -ge $TOTAL_PACKAGES ]; then
114+
exit 0
115+
fi
116+
117+
BATCH_PACKAGES=("${ALL_PACKAGES[@]:$START_INDEX:${{ env.BATCH_SIZE }}}")
118+
119+
# Strip trailing slashes to pass down directly into ci/run_conditional_tests.sh
120+
subdirs=("${BATCH_PACKAGES[@]%/}")
121+
122+
ci/run_conditional_tests.sh "${subdirs[@]}"
48123
- name: Upload coverage results
49124
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
50125
with:
51-
name: coverage-artifact-${{ matrix.python }}
126+
# Appended batch-index to separate parallel coverage uploads cleanly
127+
name: coverage-artifact-${{ matrix.python }}-${{ matrix.batch-index }}
52128
path: .coverage-${{ matrix.python }}
53129
include-hidden-files: true
54130

55131
cover:
56132
runs-on: ubuntu-latest
57133
needs:
58-
- unit
134+
- unit
59135
steps:
60136
- name: Checkout
61137
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -167,3 +243,22 @@ jobs:
167243
echo "This usually means the unit tests did not run or failed to upload their coverage files."
168244
exit 1
169245
fi
246+
247+
unittest-runtime-result:
248+
name: "unit (${{ matrix.python }})"
249+
needs: unit
250+
if: always()
251+
strategy:
252+
matrix:
253+
python: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
254+
runs-on: ubuntu-latest
255+
steps:
256+
- name: Check unit tests results
257+
run: |
258+
UNIT_STATUS="${{ needs.unit.result }}"
259+
260+
if [[ "$UNIT_STATUS" == "success" ]]; then
261+
echo "Python ${{ matrix.python }} tests passed."
262+
else
263+
echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."
264+
exit 1

‎ci/run_conditional_tests.sh‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121

2222
# `TEST_TYPE` and `PY_VERSION` are required by the script `ci/run_single_test.sh`
2323

24+
# Optional Arguments:
25+
# Pass specific space-separated package paths (e.g., "packages/google-cloud-storage") to only test those directories.
26+
# If no arguments are provided, the script automatically determines which directories have changed
27+
#
2428
# This script will determine which directories have changed
2529
# under the `packages` folder. For `BUILD_TYPE=="presubmit"`,
2630
# we'll compare against the `packages` folder in HEAD,
@@ -78,14 +82,29 @@ set -e
7882
# Now we have a fixed list, but we can change it to autodetect if
7983
# necessary.
8084

81-
subdirs=(
82-
packages
83-
)
85+
if [ $# -gt 0 ]; then
86+
subdirs=("$@")
87+
else
88+
subdirs=(
89+
packages
90+
)
91+
fi
8492

8593
RETVAL=0
8694

8795
for subdir in ${subdirs[@]}; do
88-
for d in `ls -d ${subdir}/*/`; do
96+
# If a specific package path was passed directly, use it; otherwise scan the parent folder
97+
if [ -d "${subdir}" ] && [[ "${subdir}" != "packages" ]]; then
98+
loop_dirs=("${subdir}")
99+
else
100+
loop_dirs=(`ls -d ${subdir}/*/`)
101+
fi
102+
103+
for d in "${loop_dirs[@]}"; do
104+
# Ensure the directory path always ends with a trailing slash for git diff safety
105+
if [[ "$d" != */ ]]; then
106+
d="$d/"
107+
fi
89108
should_test=false
90109
if [ -n "${GIT_DIFF_ARG}" ]; then
91110
echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"

0 commit comments

Comments
 (0)