Skip to content

Commit 0c053e3

Browse files
hjmjohnsonclaude
hjmjohnson
and
claude
committed
ENH: Limit CI build and test parallelism to match available vCPUs
Oversubscribing parallel jobs on CI runners causes several problems: - ITK's template-heavy C++ compilations each consume 1-2 GB of RAM; launching more parallel compiler instances than vCPUs on memory-constrained runners (e.g., 4 Ninja jobs on a 2-vCPU/7 GB Azure agent) risks OOM kills and thrashing that slow the build. - CI VMs share physical hosts with other tenants, so excess parallelism contends for shared CPU, cache, and memory bandwidth, yielding diminishing returns and unpredictable wall-clock times. - Context-switch overhead on oversubscribed vCPUs wastes cycles on scheduling rather than compilation, increasing overall build time. Set CTEST_BUILD_FLAGS and ctest -j to the actual vCPU count for each runner: 2 for Azure Standard_DS2_v2 (7 GB), 3 for Azure/GitHub macOS-15 (14 GB), and 4 for GitHub ubuntu-24.04-arm (16 GB). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 36140b2 commit 0c053e3

8 files changed

Lines changed: 23 additions & 10 deletions

‎.github/workflows/arm.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ jobs:
4747
cmake-build-type: "Release"
4848
cmake-generator: "Ninja"
4949
python-version: ""
50+
parallel-level: 4
5051
ctest-cache: |
5152
BUILD_SHARED_LIBS:BOOL=OFF
5253
BUILD_EXAMPLES:BOOL=OFF
@@ -60,6 +61,7 @@ jobs:
6061
cmake-build-type: "Release"
6162
cmake-generator: "Ninja"
6263
python-version: ""
64+
parallel-level: 3
6365
ctest-cache: |
6466
BUILD_SHARED_LIBS:BOOL=OFF
6567
BUILD_EXAMPLES:BOOL=OFF
@@ -74,6 +76,7 @@ jobs:
7476
cmake-build-type: "Release"
7577
cmake-generator: "Ninja"
7678
python-version: "3.11"
79+
parallel-level: 3
7780
ctest-cache: |
7881
BUILD_SHARED_LIBS:BOOL=OFF
7982
BUILD_EXAMPLES:BOOL=OFF
@@ -155,6 +158,7 @@ jobs:
155158
cat > ITK-dashboard/dashboard.cmake << EOF
156159
set(CTEST_BUILD_CONFIGURATION "${{ matrix.cmake-build-type }}")
157160
set(CTEST_CMAKE_GENERATOR "${{ matrix.cmake-generator }}")
161+
set(CTEST_BUILD_FLAGS "-j ${{ matrix.parallel-level }}")
158162
set(dashboard_cache "
159163
${{ matrix.ctest-cache }}
160164
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
@@ -171,7 +175,7 @@ jobs:
171175
c++ --version
172176
cmake --version
173177
174-
ctest -S ${{ github.workspace }}/ITK-dashboard/dashboard.cmake -VV -j 4 ${{ matrix.ctest-options }}
178+
ctest -S ${{ github.workspace }}/ITK-dashboard/dashboard.cmake -VV -j ${{ matrix.parallel-level }} ${{ matrix.ctest-options }}
175179
env:
176180
CTEST_OUTPUT_ON_FAILURE: 1
177181
- name: Save compiler cache

‎Testing/ContinuousIntegration/AzurePipelinesBatch.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ jobs:
8585
set(CTEST_CMAKE_GENERATOR_TOOLSET "$(CTEST_CMAKE_GENERATOR_TOOLSET)")
8686
set(CTEST_CMAKE_GENERATOR_PLATFORM "$(CTEST_CMAKE_GENERATOR_PLATFORM)")
8787
set(CTEST_CMAKE_GENERATOR "$(CTEST_CMAKE_GENERATOR)")
88+
set(CTEST_BUILD_FLAGS "/m:2")
8889
set(dashboard_cache "
8990
BUILD_SHARED_LIBS:BOOL=ON
9091
BUILD_EXAMPLES:BOOL=OFF
@@ -103,7 +104,7 @@ jobs:
103104
- script: |
104105
cmake --version
105106
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
106-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
107+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2
107108
displayName: 'Build and test'
108109
env:
109110
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesLinux.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ jobs:
8989
cat > dashboard.cmake << EOF
9090
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
9191
set(CTEST_CMAKE_GENERATOR "Ninja")
92+
set(CTEST_BUILD_FLAGS "-j 2")
9293
set(dashboard_cache "
9394
BUILD_SHARED_LIBS:BOOL=OFF
9495
BUILD_EXAMPLES:BOOL=OFF
@@ -111,7 +112,7 @@ jobs:
111112
c++ --version
112113
cmake --version
113114
114-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
115+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2
115116
displayName: 'Build and test'
116117
env:
117118
CTEST_OUTPUT_ON_FAILURE: 1
@@ -179,6 +180,7 @@ jobs:
179180
cat > dashboard.cmake << EOF
180181
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
181182
set(CTEST_CMAKE_GENERATOR "Ninja")
183+
set(CTEST_BUILD_FLAGS "-j 2")
182184
set(BUILD_NAME_SUFFIX "LegacyRemoved")
183185
set(dashboard_cache "
184186
ITK_LEGACY_REMOVE:BOOL=ON
@@ -202,7 +204,7 @@ jobs:
202204
c++ --version
203205
cmake --version
204206
205-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
207+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2
206208
displayName: 'Build and test'
207209
env:
208210
CTEST_OUTPUT_ON_FAILURE: 1
@@ -270,6 +272,7 @@ jobs:
270272
cat > dashboard.cmake << EOF
271273
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
272274
set(CTEST_CMAKE_GENERATOR "Ninja")
275+
set(CTEST_BUILD_FLAGS "-j 2")
273276
set(BUILD_NAME_SUFFIX "Cxx20")
274277
set(dashboard_cache "
275278
CMAKE_CXX_STANDARD:STRING=20
@@ -291,7 +294,7 @@ jobs:
291294
set -x
292295
c++ --version
293296
cmake --version
294-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
297+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2
295298
displayName: "Build and test"
296299
env:
297300
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ jobs:
9393
cat > dashboard.cmake << EOF
9494
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
9595
set(CTEST_CMAKE_GENERATOR "Ninja")
96+
set(CTEST_BUILD_FLAGS "-j 2")
9697
set(BUILD_NAME_SUFFIX "-Python")
9798
set(dashboard_cache "
9899
Python3_ROOT_DIR:PATH=${python_root_dir}
@@ -118,7 +119,7 @@ jobs:
118119
c++ --version
119120
cmake --version
120121
121-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4 -L Python -E "(PythonExtrasTest)|(PythonFastMarching)|(PythonLazyLoadingImage)|(PythonThresholdSegmentationLevelSetWhiteMatterTest)|(PythonVerifyTTypeAPIConsistency)|(PythonWatershedSegmentation1Test)"
122+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2 -L Python -E "(PythonExtrasTest)|(PythonFastMarching)|(PythonLazyLoadingImage)|(PythonThresholdSegmentationLevelSetWhiteMatterTest)|(PythonVerifyTTypeAPIConsistency)|(PythonWatershedSegmentation1Test)"
122123
displayName: 'Build and test'
123124
env:
124125
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesMacOS.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ jobs:
8989
cat > dashboard.cmake << EOF
9090
set(CTEST_BUILD_CONFIGURATION "Release")
9191
set(CTEST_CMAKE_GENERATOR "Ninja")
92+
set(CTEST_BUILD_FLAGS "-j 3")
9293
set(dashboard_cache "
9394
BUILD_SHARED_LIBS:BOOL=ON
9495
BUILD_EXAMPLES:BOOL=ON
@@ -110,7 +111,7 @@ jobs:
110111
c++ --version
111112
cmake --version
112113
113-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
114+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 3
114115
displayName: 'Build and test'
115116
env:
116117
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ jobs:
9494
cat > dashboard.cmake << EOF
9595
set(CTEST_BUILD_CONFIGURATION "Release")
9696
set(CTEST_CMAKE_GENERATOR "Ninja")
97+
set(CTEST_BUILD_FLAGS "-j 3")
9798
set(BUILD_NAME_SUFFIX "-Python")
9899
set(dashboard_cache "
99100
Python3_ROOT_DIR:PATH=${python_root_dir}
@@ -119,7 +120,7 @@ jobs:
119120
c++ --version
120121
cmake --version
121122
122-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
123+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 3
123124
displayName: 'Build and test'
124125
env:
125126
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesWindows.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ jobs:
8484
cat > dashboard.cmake << EOF
8585
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
8686
set(CTEST_CMAKE_GENERATOR "Ninja")
87+
set(CTEST_BUILD_FLAGS "-j 2")
8788
set(ENV{CC} cl.exe)
8889
set(ENV{CXX} cl.exe)
8990
set(dashboard_cache "
@@ -104,7 +105,7 @@ jobs:
104105
- script: |
105106
cmake --version
106107
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
107-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4
108+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2
108109
displayName: 'Build and test'
109110
env:
110111
CTEST_OUTPUT_ON_FAILURE: 1

‎Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ jobs:
8585
cat > dashboard.cmake << EOF
8686
set(CTEST_BUILD_CONFIGURATION "MinSizeRel")
8787
set(CTEST_CMAKE_GENERATOR "Ninja")
88+
set(CTEST_BUILD_FLAGS "-j 2")
8889
set(BUILD_NAME_SUFFIX "-Python")
8990
set(ENV{CC} cl.exe)
9091
set(ENV{CXX} cl.exe)
@@ -114,7 +115,7 @@ jobs:
114115
- script: |
115116
cmake --version
116117
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
117-
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 4 -L Python
118+
ctest -S $(Agent.BuildDirectory)/ITK-dashboard/dashboard.cmake -VV -j 2 -L Python
118119
displayName: 'Build and test'
119120
env:
120121
CTEST_OUTPUT_ON_FAILURE: 1

0 commit comments

Comments
 (0)