Skip to content

Commit f69ecb5

Browse files
vladimavladima
committed
run tests in parallel by equally dividing them between workers
1 parent 92d465d commit f69ecb5

8 files changed

Lines changed: 246 additions & 49 deletions

File tree

‎Jakefile.js‎

Lines changed: 69 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -680,9 +680,9 @@ function cleanTestDirs() {
680680
}
681681

682682
// used to pass data from jake command line directly to run.js
683-
function writeTestConfigFile(tests, light, testConfigFile) {
684-
console.log('Running test(s): ' + tests);
685-
var testConfigContents = JSON.stringify({ test: [tests], light: light });
683+
function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile) {
684+
var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder });
685+
console.log('Running tests with config: ' + testConfigContents);
686686
fs.writeFileSync('test.config', testConfigContents);
687687
}
688688

@@ -692,7 +692,7 @@ function deleteTemporaryProjectOutput() {
692692
}
693693
}
694694

695-
function runConsoleTests(defaultReporter, defaultSubsets) {
695+
function runConsoleTests(defaultReporter, runInParallel) {
696696
cleanTestDirs();
697697
var debug = process.env.debug || process.env.d;
698698
tests = process.env.test || process.env.tests || process.env.t;
@@ -701,9 +701,22 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
701701
if(fs.existsSync(testConfigFile)) {
702702
fs.unlinkSync(testConfigFile);
703703
}
704+
var workerCount, taskConfigsFolder;
705+
if (runInParallel) {
706+
// generate name to store task configuration files
707+
var prefix = os.tmpdir() + "/ts-tests";
708+
var i = 1;
709+
do {
710+
taskConfigsFolder = prefix + i;
711+
i++;
712+
} while (fs.existsSync(taskConfigsFolder));
713+
fs.mkdirSync(taskConfigsFolder);
714+
715+
workerCount = process.env.workerCount || os.cpus().length;
716+
}
704717

705-
if (tests || light) {
706-
writeTestConfigFile(tests, light, testConfigFile);
718+
if (tests || light || taskConfigsFolder) {
719+
writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile);
707720
}
708721

709722
if (tests && tests.toLocaleLowerCase() === "rwc") {
@@ -717,45 +730,71 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
717730

718731
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
719732
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
720-
var subsetRegexes;
721-
if(defaultSubsets.length === 0) {
722-
subsetRegexes = [tests];
723-
}
724-
else {
725-
var subsets = tests ? tests.split("|") : defaultSubsets;
726-
subsetRegexes = subsets.map(function (sub) { return "^" + sub + ".*$"; });
727-
subsetRegexes.push("^(?!" + subsets.join("|") + ").*$");
728-
}
729-
subsetRegexes.forEach(function (subsetRegex, i) {
730-
tests = subsetRegex ? ' -g "' + subsetRegex + '"' : '';
733+
if(!runInParallel) {
734+
tests = tests ? ' -g "' + tests + '"' : '';
731735
var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
732736
console.log(cmd);
733-
function finish() {
734-
deleteTemporaryProjectOutput();
735-
complete();
736-
}
737737
exec(cmd, function () {
738-
if (lintFlag && i === 0) {
739-
var lint = jake.Task['lint'];
740-
lint.addListener('complete', function () {
741-
complete();
742-
});
743-
lint.invoke();
738+
if (i === 0) {
739+
runLinter();
744740
}
745741
finish();
746742
}, finish);
747-
});
743+
744+
}
745+
else {
746+
// run task to load all tests and partition then between workers
747+
var cmd = "mocha " + " -R min " + colors + run;
748+
console.log(cmd);
749+
exec(cmd, function() {
750+
// read all configuration files and spawn a worker for every config
751+
var configFiles = fs.readdirSync(taskConfigsFolder);
752+
var counter = configFiles.length;
753+
// schedule work for chunks
754+
configFiles.forEach(function (f) {
755+
var configPath = path.join(taskConfigsFolder, f);
756+
var workerCmd = "mocha" + " -t " + testTimeout + " -R " + reporter + " " + colors + " " + run + " --config='" + configPath + "'";
757+
console.log(workerCmd);
758+
exec(workerCmd, finishWorker, finishWorker)
759+
});
760+
761+
function finishWorker() {
762+
counter--;
763+
if (counter === 0) {
764+
// last worker clean everything and runs linter
765+
runLinter();
766+
deleteTemporaryProjectOutput();
767+
jake.rmRf(taskConfigsFolder);
768+
}
769+
complete();
770+
}
771+
});
772+
}
773+
function finish() {
774+
deleteTemporaryProjectOutput();
775+
complete();
776+
}
777+
function runLinter() {
778+
if (!lintFlag) {
779+
return;
780+
}
781+
var lint = jake.Task['lint'];
782+
lint.addListener('complete', function () {
783+
complete();
784+
});
785+
lint.invoke();
786+
}
748787
}
749788

750789
var testTimeout = 20000;
751790
desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
752791
task("runtests-parallel", ["build-rules", "tests", builtLocalDirectory], function() {
753-
runConsoleTests('min', ['compiler', 'conformance', 'Projects', 'fourslash']);
792+
runConsoleTests('min', /*runInParallel*/ true);
754793
}, {async: true});
755794

756795
desc("Runs the tests using the built run.js file. Optional arguments are: t[ests]=regex r[eporter]=[list|spec|json|<more>] d[ebug]=true color[s]=false lint=true.");
757796
task("runtests", ["build-rules", "tests", builtLocalDirectory], function() {
758-
runConsoleTests('mocha-fivemat-progress-reporter', []);
797+
runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false);
759798
}, {async: true});
760799

761800
desc("Generates code coverage data via instanbul");

‎src/harness/compilerRunner.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ const enum CompilerTestType {
1111

1212
class CompilerBaselineRunner extends RunnerBase {
1313
private basePath = "tests/cases";
14-
private testSuiteName: string;
14+
private testSuiteName: TestRunnerKind;
1515
private errors: boolean;
1616
private emit: boolean;
1717
private decl: boolean;
@@ -40,6 +40,14 @@ class CompilerBaselineRunner extends RunnerBase {
4040
this.basePath += "/" + this.testSuiteName;
4141
}
4242

43+
public kind() {
44+
return this.testSuiteName;
45+
}
46+
47+
public enumerateTestFiles() {
48+
return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
49+
}
50+
4351
private makeUnitName(name: string, root: string) {
4452
return ts.isRootedDiskPath(name) ? name : ts.combinePaths(root, name);
4553
};
@@ -390,7 +398,7 @@ class CompilerBaselineRunner extends RunnerBase {
390398

391399
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
392400
if (this.tests.length === 0) {
393-
const testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
401+
const testFiles = this.enumerateTestFiles();
394402
testFiles.forEach(fn => {
395403
fn = fn.replace(/\\/g, "/");
396404
this.checkTestCodeOutput(fn);

‎src/harness/fourslashRunner.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const enum FourSlashTestType {
1212

1313
class FourSlashRunner extends RunnerBase {
1414
protected basePath: string;
15-
protected testSuiteName: string;
15+
protected testSuiteName: TestRunnerKind;
1616

1717
constructor(private testType: FourSlashTestType) {
1818
super();
@@ -36,9 +36,17 @@ class FourSlashRunner extends RunnerBase {
3636
}
3737
}
3838

39+
public enumerateTestFiles() {
40+
return this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
41+
}
42+
43+
public kind() {
44+
return this.testSuiteName;
45+
}
46+
3947
public initializeTests() {
4048
if (this.tests.length === 0) {
41-
this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
49+
this.tests = this.enumerateTestFiles();
4250
}
4351

4452
describe(this.testSuiteName + " tests", () => {

‎src/harness/projectsRunner.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,19 @@ interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult {
3737
}
3838

3939
class ProjectRunner extends RunnerBase {
40+
41+
public enumerateTestFiles() {
42+
return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
43+
}
44+
45+
public kind(): TestRunnerKind {
46+
return "project";
47+
}
48+
4049
public initializeTests() {
4150
if (this.tests.length === 0) {
42-
const testFiles = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
51+
const testFiles = this.enumerateTestFiles();
4352
testFiles.forEach(fn => {
44-
fn = fn.replace(/\\/g, "/");
4553
this.runProjectTestCase(fn);
4654
});
4755
}

‎src/harness/runner.ts‎

Lines changed: 118 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,18 +33,90 @@ function runTests(runners: RunnerBase[]) {
3333
}
3434
}
3535

36+
function tryGetConfig(args: string[]) {
37+
const prefix = "--config=";
38+
const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length));
39+
// strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically)
40+
return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, "");
41+
}
42+
43+
function createRunner(kind: TestRunnerKind): RunnerBase {
44+
switch (kind) {
45+
case "conformance":
46+
return new CompilerBaselineRunner(CompilerTestType.Conformance);
47+
case "compiler":
48+
return new CompilerBaselineRunner(CompilerTestType.Regressions);
49+
case "fourslash":
50+
return new FourSlashRunner(FourSlashTestType.Native);
51+
case "fourslash-shims":
52+
return new FourSlashRunner(FourSlashTestType.Shims);
53+
case "fourslash-shims-pp":
54+
return new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess);
55+
case "fourslash-server":
56+
return new FourSlashRunner(FourSlashTestType.Server);
57+
case "project":
58+
return new ProjectRunner();
59+
case "rwc":
60+
return new RWCRunner();
61+
case "test262":
62+
return new Test262BaselineRunner();
63+
}
64+
}
65+
3666
// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options
37-
let mytestconfig = "mytest.config";
38-
let testconfig = "test.config";
39-
let testConfigFile =
40-
Harness.IO.fileExists(mytestconfig) ? Harness.IO.readFile(mytestconfig) :
41-
(Harness.IO.fileExists(testconfig) ? Harness.IO.readFile(testconfig) : "");
42-
43-
if (testConfigFile !== "") {
44-
const testConfig = JSON.parse(testConfigFile);
67+
68+
const mytestconfigFileName = "mytest.config";
69+
const testconfigFileName = "test.config";
70+
71+
const customConfig = tryGetConfig(Harness.IO.args());
72+
let testConfigContent =
73+
customConfig && Harness.IO.fileExists(customConfig)
74+
? Harness.IO.readFile(customConfig)
75+
: Harness.IO.fileExists(mytestconfigFileName)
76+
? Harness.IO.readFile(mytestconfigFileName)
77+
: Harness.IO.fileExists(testconfigFileName) ? Harness.IO.readFile(testconfigFileName) : "";
78+
79+
let taskConfigsFolder: string;
80+
let workerCount: number;
81+
let runUnitTests = true;
82+
83+
interface TestConfig {
84+
light?: boolean;
85+
taskConfigsFolder?: string;
86+
workerCount?: number;
87+
tasks?: TaskSet[];
88+
test?: string[];
89+
runUnitTests?: boolean;
90+
}
91+
92+
interface TaskSet {
93+
runner: TestRunnerKind;
94+
files: string[];
95+
}
96+
97+
if (testConfigContent !== "") {
98+
const testConfig = <TestConfig>JSON.parse(testConfigContent);
4599
if (testConfig.light) {
46100
Harness.lightMode = true;
47101
}
102+
if (testConfig.taskConfigsFolder) {
103+
taskConfigsFolder = testConfig.taskConfigsFolder;
104+
}
105+
if (testConfig.runUnitTests !== undefined) {
106+
runUnitTests = testConfig.runUnitTests;
107+
}
108+
if (testConfig.workerCount) {
109+
workerCount = testConfig.workerCount;
110+
}
111+
if (testConfig.tasks) {
112+
for (const taskSet of testConfig.tasks) {
113+
const runner = createRunner(taskSet.runner);
114+
for (const file of taskSet.files) {
115+
runner.addTest(file);
116+
}
117+
runners.push(runner);
118+
}
119+
}
48120

49121
if (testConfig.test && testConfig.test.length > 0) {
50122
for (const option of testConfig.test) {
@@ -108,4 +180,41 @@ if (runners.length === 0) {
108180
// runners.push(new GeneratedFourslashRunner());
109181
}
110182

111-
runTests(runners);
183+
if (taskConfigsFolder) {
184+
// this instance of mocha should only partition work but not run actual tests
185+
runUnitTests = false;
186+
const workerConfigs: TestConfig[] = [];
187+
for (let i = 0; i < workerCount; i++) {
188+
// pass light mode settings to workers
189+
workerConfigs.push({ light: Harness.lightMode, tasks: [] });
190+
}
191+
192+
for (const runner of runners) {
193+
const files = runner.enumerateTestFiles();
194+
const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding
195+
for (let i = 0; i < workerCount; i++) {
196+
const startPos = i * chunkSize;
197+
const len = Math.min(chunkSize, files.length - startPos);
198+
if (len !== 0) {
199+
workerConfigs[i].tasks.push({
200+
runner: runner.kind(),
201+
files: files.slice(startPos, startPos + len)
202+
});
203+
}
204+
}
205+
}
206+
207+
for (let i = 0; i < workerCount; i++) {
208+
const config = workerConfigs[i];
209+
// use last worker to run unit tests
210+
config.runUnitTests = i === workerCount - 1;
211+
Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i]));
212+
}
213+
}
214+
else {
215+
runTests(runners);
216+
}
217+
if (!runUnitTests) {
218+
// patch `describe` to skip unit tests
219+
describe = <any>describe.skip;
220+
}

‎src/harness/runnerbase.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
/// <reference path="harness.ts" />
22

3+
4+
type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262";
5+
type CompilerTestKind = "conformance" | "compiler";
6+
type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
7+
38
abstract class RunnerBase {
49
constructor() { }
510

@@ -12,9 +17,13 @@ abstract class RunnerBase {
1217
}
1318

1419
public enumerateFiles(folder: string, regex?: RegExp, options?: { recursive: boolean }): string[] {
15-
return Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) });
20+
return ts.map(Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }), ts.normalizeSlashes);
1621
}
1722

23+
abstract kind(): TestRunnerKind;
24+
25+
abstract enumerateTestFiles(): string[];
26+
1827
/** Setup the runner's tests so that they are ready to be executed by the harness
1928
* The first test should be a describe/it block that sets up the harness's compiler instance appropriately
2029
*/

0 commit comments

Comments
 (0)