Skip to content

Commit 84e3681

Browse files
weswighamweswigham
authored
Support timeouts in the parallel runner (microsoft#20631)
* Support timeouts in the parallel runner * Apply PR feedback: unify code paths, use string as sentinel
1 parent b5fda49 commit 84e3681

6 files changed

Lines changed: 106 additions & 29 deletions

File tree

‎Gulpfile.ts‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -680,14 +680,14 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done:
680680
workerCount = cmdLineOptions.workers;
681681
}
682682

683-
if (tests || runners || light || taskConfigsFolder) {
684-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit);
685-
}
686-
687683
if (tests && tests.toLocaleLowerCase() === "rwc") {
688684
testTimeout = 400000;
689685
}
690686

687+
if (tests || runners || light || testTimeout || taskConfigsFolder) {
688+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
689+
}
690+
691691
const colors = cmdLineOptions.colors;
692692
const reporter = cmdLineOptions.reporter || defaultReporter;
693693

@@ -872,8 +872,17 @@ function cleanTestDirs(done: (e?: any) => void) {
872872
}
873873

874874
// used to pass data from jake command line directly to run.js
875-
function writeTestConfigFile(tests: string, runners: string, light: boolean, taskConfigsFolder?: string, workerCount?: number, stackTraceLimit?: string) {
876-
const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, runner: runners ? runners.split(",") : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder, noColor: !cmdLineOptions.colors });
875+
function writeTestConfigFile(tests: string, runners: string, light: boolean, taskConfigsFolder?: string, workerCount?: number, stackTraceLimit?: string, timeout?: number) {
876+
const testConfigContents = JSON.stringify({
877+
test: tests ? [tests] : undefined,
878+
runner: runners ? runners.split(",") : undefined,
879+
light,
880+
workerCount,
881+
stackTraceLimit,
882+
taskConfigsFolder,
883+
noColor: !cmdLineOptions.colors,
884+
timeout,
885+
});
877886
console.log("Running tests with config: " + testConfigContents);
878887
fs.writeFileSync("test.config", testConfigContents);
879888
}

‎Jakefile.js‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -858,15 +858,16 @@ function cleanTestDirs() {
858858
}
859859

860860
// used to pass data from jake command line directly to run.js
861-
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors) {
861+
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) {
862862
var testConfigContents = JSON.stringify({
863863
runners: runners ? runners.split(",") : undefined,
864864
test: tests ? [tests] : undefined,
865865
light: light,
866866
workerCount: workerCount,
867867
taskConfigsFolder: taskConfigsFolder,
868868
stackTraceLimit: stackTraceLimit,
869-
noColor: !colors
869+
noColor: !colors,
870+
timeout: testTimeout
870871
});
871872
fs.writeFileSync('test.config', testConfigContents);
872873
}
@@ -908,14 +909,14 @@ function runConsoleTests(defaultReporter, runInParallel) {
908909
workerCount = process.env.workerCount || process.env.p || os.cpus().length;
909910
}
910911

911-
if (tests || runners || light || taskConfigsFolder) {
912-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors);
913-
}
914-
915912
if (tests && tests.toLocaleLowerCase() === "rwc") {
916913
testTimeout = 800000;
917914
}
918915

916+
if (tests || runners || light || testTimeout || taskConfigsFolder) {
917+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout);
918+
}
919+
919920
var colorsFlag = process.env.color || process.env.colors;
920921
var colors = colorsFlag !== "false" && colorsFlag !== "0";
921922
var reporter = process.env.reporter || process.env.r || defaultReporter;

‎src/harness/parallel/host.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ namespace Harness.Parallel.Host {
99
on(event: "error", listener: (err: Error) => void): this;
1010
on(event: "exit", listener: (code: number, signal: string) => void): this;
1111
on(event: "message", listener: (message: ParallelClientMessage) => void): this;
12+
kill(signal?: string): void;
13+
currentTasks?: {file: string}[]; // Custom monkeypatch onto child process handle
1214
}
1315

1416
interface ProgressBarsOptions {
@@ -134,13 +136,26 @@ namespace Harness.Parallel.Host {
134136
const newPerfData: {[testHash: string]: number} = {};
135137

136138
const workers: ChildProcessPartial[] = [];
139+
const defaultTimeout = globalTimeout !== undefined
140+
? globalTimeout
141+
: mocha && mocha.suite && mocha.suite._timeout
142+
? mocha.suite._timeout
143+
: 20000; // 20 seconds
137144
let closedWorkers = 0;
138145
for (let i = 0; i < workerCount; i++) {
139146
// TODO: Just send the config over the IPC channel or in the command line arguments
140147
const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests };
141148
const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`);
142149
Harness.IO.writeFile(configPath, JSON.stringify(config));
143150
const child = fork(__filename, [`--config="${configPath}"`]);
151+
let currentTimeout = defaultTimeout;
152+
const killChild = () => {
153+
child.kill();
154+
console.error(`Worker exceeded timeout ${child.currentTasks && child.currentTasks.length ? `while running test '${child.currentTasks[0].file}'.` : `during test setup.`}`);
155+
return process.exit(2);
156+
};
157+
let timer = setTimeout(killChild, currentTimeout);
158+
const timeoutStack: number[] = [];
144159
child.on("error", err => {
145160
console.error("Unexpected error in child process:");
146161
console.error(err);
@@ -160,8 +175,23 @@ namespace Harness.Parallel.Host {
160175
Stack: ${data.payload.stack}`);
161176
return process.exit(2);
162177
}
178+
case "timeout": {
179+
if (data.payload.duration === "reset") {
180+
currentTimeout = timeoutStack.pop() || defaultTimeout;
181+
}
182+
else {
183+
timeoutStack.push(currentTimeout);
184+
currentTimeout = data.payload.duration;
185+
}
186+
break;
187+
}
163188
case "progress":
164189
case "result": {
190+
clearTimeout(timer);
191+
timer = setTimeout(killChild, currentTimeout);
192+
if (child.currentTasks) {
193+
child.currentTasks.shift();
194+
}
165195
totalPassing += data.payload.passing;
166196
if (data.payload.errors.length) {
167197
errorResults = errorResults.concat(data.payload.errors);
@@ -195,6 +225,7 @@ namespace Harness.Parallel.Host {
195225
while (tasks.length && taskList.reduce((p, c) => p + c.size, 0) < chunkSize) {
196226
taskList.push(tasks.pop());
197227
}
228+
child.currentTasks = taskList;
198229
if (taskList.length === 1) {
199230
child.send({ type: "test", payload: taskList[0] });
200231
}
@@ -252,18 +283,22 @@ namespace Harness.Parallel.Host {
252283
for (const worker of workers) {
253284
const payload = batches.pop();
254285
if (payload) {
286+
worker.currentTasks = payload;
255287
worker.send({ type: "batch", payload });
256288
}
257289
else { // Out of batches, send off just one test
258290
const payload = tasks.pop();
259291
ts.Debug.assert(!!payload); // The reserve kept above should ensure there is always an initial task available, even in suboptimal scenarios
292+
worker.currentTasks = [payload];
260293
worker.send({ type: "test", payload });
261294
}
262295
}
263296
}
264297
else {
265298
for (let i = 0; i < workerCount; i++) {
266-
workers[i].send({ type: "test", payload: tasks.pop() });
299+
const task = tasks.pop();
300+
workers[i].currentTasks = [task];
301+
workers[i].send({ type: "test", payload: task });
267302
}
268303
}
269304

‎src/harness/parallel/shared.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ namespace Harness.Parallel {
1010
export type ErrorInfo = ParallelErrorMessage["payload"] & { name: string[] };
1111
export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[], duration: number, runner: TestRunnerKind | "unittest", file: string } } | never;
1212
export type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never;
13-
export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage;
13+
export type ParallelTimeoutChangeMessage = { type: "timeout", payload: { duration: number | "reset" } } | never;
14+
export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage | ParallelTimeoutChangeMessage;
1415
}

‎src/harness/parallel/worker.ts‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,28 @@ namespace Harness.Parallel.Worker {
3636
}) as Mocha.ITestDefinition;
3737
}
3838

39+
function setTimeoutAndExecute(timeout: number | undefined, f: () => void) {
40+
if (timeout !== undefined) {
41+
const timeoutMsg: ParallelTimeoutChangeMessage = { type: "timeout", payload: { duration: timeout } };
42+
process.send(timeoutMsg);
43+
}
44+
f();
45+
if (timeout !== undefined) {
46+
// Reset timeout
47+
const timeoutMsg: ParallelTimeoutChangeMessage = { type: "timeout", payload: { duration: "reset" } };
48+
process.send(timeoutMsg);
49+
}
50+
}
51+
3952
function executeSuiteCallback(name: string, callback: MochaCallback) {
53+
let timeout: number;
4054
const fakeContext: Mocha.ISuiteCallbackContext = {
4155
retries() { return this; },
4256
slow() { return this; },
43-
timeout() { return this; },
57+
timeout(n) {
58+
timeout = n;
59+
return this;
60+
},
4461
};
4562
namestack.push(name);
4663
let beforeFunc: Callable;
@@ -71,7 +88,10 @@ namespace Harness.Parallel.Worker {
7188
finally {
7289
beforeFunc = undefined;
7390
}
74-
testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
91+
92+
setTimeoutAndExecute(timeout, () => {
93+
testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
94+
});
7595

7696
try {
7797
if (afterFunc) {
@@ -103,9 +123,13 @@ namespace Harness.Parallel.Worker {
103123
}
104124

105125
function executeTestCallback(name: string, callback: MochaCallback) {
126+
let timeout: number;
106127
const fakeContext: Mocha.ITestCallbackContext = {
107128
skip() { return this; },
108-
timeout() { return this; },
129+
timeout(n) {
130+
timeout = n;
131+
return this;
132+
},
109133
retries() { return this; },
110134
slow() { return this; },
111135
};
@@ -121,18 +145,20 @@ namespace Harness.Parallel.Worker {
121145
}
122146
}
123147
if (callback.length === 0) {
124-
try {
125-
// TODO: If we ever start using async test completions, polyfill promise return handling
126-
callback.call(fakeContext);
127-
}
128-
catch (error) {
129-
errors.push({ error: error.message, stack: error.stack, name: [...namestack] });
130-
return;
131-
}
132-
finally {
133-
namestack.pop();
134-
}
135-
passing++;
148+
setTimeoutAndExecute(timeout, () => {
149+
try {
150+
// TODO: If we ever start using async test completions, polyfill promise return handling
151+
callback.call(fakeContext);
152+
}
153+
catch (error) {
154+
errors.push({ error: error.message, stack: error.stack, name: [...namestack] });
155+
return;
156+
}
157+
finally {
158+
namestack.pop();
159+
}
160+
passing++;
161+
});
136162
}
137163
else {
138164
// Uses `done` callback

‎src/harness/runner.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ interface TestConfig {
100100
runners?: string[];
101101
runUnitTests?: boolean;
102102
noColors?: boolean;
103+
timeout?: number;
103104
}
104105

105106
interface TaskSet {
@@ -108,12 +109,16 @@ interface TaskSet {
108109
}
109110

110111
let configOption: string;
112+
let globalTimeout: number;
111113
function handleTestConfig() {
112114
if (testConfigContent !== "") {
113115
const testConfig = <TestConfig>JSON.parse(testConfigContent);
114116
if (testConfig.light) {
115117
Harness.lightMode = true;
116118
}
119+
if (testConfig.timeout) {
120+
globalTimeout = testConfig.timeout;
121+
}
117122
runUnitTests = testConfig.runUnitTests;
118123
if (testConfig.workerCount) {
119124
workerCount = +testConfig.workerCount;

0 commit comments

Comments
 (0)