Reduce test time by lowering the sample count for perf tests (#1888)

DonJayamanne · web-flow · commit 870dbfe9dc0e · 2018-06-06T19:45:01.000-07:00
diff --git a/news/3 Code Health/1887.md b/news/3 Code Health/1887.md
@@ -0,0 +1 @@
+Reduce sample count used to capture performance metrics in order to reduce time taken to complete the tests.
diff --git a/src/test/performance/load.perf.test.ts b/src/test/performance/load.perf.test.ts
@@ -18,7 +18,7 @@ suite('Activation Times', () => {
     if (process.env.ACTIVATION_TIMES_LOG_FILE_PATH) {
         const logFile = process.env.ACTIVATION_TIMES_LOG_FILE_PATH;
         const sampleCounter = fs.existsSync(logFile) ? fs.readFileSync(logFile, { encoding: 'utf8' }).toString().split(/\r?\n/g).length : 1;
-        if (sampleCounter > 10) {
+        if (sampleCounter > 5) {
             return;
         }
         test(`Capture Extension Activation Times (Version: ${process.env.ACTIVATION_TIMES_EXT_VERSION}, sample: ${sampleCounter})`, async () => {
diff --git a/src/test/performanceTest.ts b/src/test/performanceTest.ts
@@ -38,7 +38,7 @@ class TestRunner {
         await del([path.join(tmpFolder, '**')]);
         await this.extractLatestExtension(publishedExtensionPath);
 
-        const timesToLoadEachVersion = 3;
+        const timesToLoadEachVersion = 2;
         const devLogFiles: string[] = [];
         const releaseLogFiles: string[] = [];
         const newAnalysisEngineLogFiles: string[] = [];
@@ -47,20 +47,24 @@ class TestRunner {
             await this.enableNewAnalysisEngine(false);
 
             const devLogFile = path.join(logFilesPath, `dev_loadtimes${i}.txt`);
+            console.log(`Start Performance Tests: Counter ${i}, for Dev version with Jedi`);
             await this.capturePerfTimes(Version.Dev, devLogFile);
             devLogFiles.push(devLogFile);
 
             const releaseLogFile = path.join(logFilesPath, `release_loadtimes${i}.txt`);
+            console.log(`Start Performance Tests: Counter ${i}, for Release version with Jedi`);
             await this.capturePerfTimes(Version.Release, releaseLogFile);
             releaseLogFiles.push(releaseLogFile);
 
             // New Analysis engine.
             await this.enableNewAnalysisEngine(true);
             const newAnalysisEngineLogFile = path.join(logFilesPath, `newAnalysisEngine_loadtimes${i}.txt`);
+            console.log(`Start Performance Tests: Counter ${i}, for Release version with Analysis Engine`);
             await this.capturePerfTimes(Version.Release, newAnalysisEngineLogFile);
             newAnalysisEngineLogFiles.push(newAnalysisEngineLogFile);
         }
 
+        console.log('Compare Performance Results');
         await this.runPerfTest(devLogFiles, releaseLogFiles, newAnalysisEngineLogFiles);
     }
     private async enableNewAnalysisEngine(enable: boolean) {

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Reduce sample count used to capture performance metrics in order to reduce time taken to complete the tests.`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ suite('Activation Times', () => {`
`18`	`18`	`if (process.env.ACTIVATION_TIMES_LOG_FILE_PATH) {`
`19`	`19`	`const logFile = process.env.ACTIVATION_TIMES_LOG_FILE_PATH;`
`20`	`20`	`const sampleCounter = fs.existsSync(logFile) ? fs.readFileSync(logFile, { encoding: 'utf8' }).toString().split(/\r?\n/g).length : 1;`
`21`		`- if (sampleCounter > 10) {`
	`21`	`+ if (sampleCounter > 5) {`
`22`	`22`	`return;`
`23`	`23`	`}`
`24`	`24`	test(`Capture Extension Activation Times (Version: ${process.env.ACTIVATION_TIMES_EXT_VERSION}, sample: ${sampleCounter})`, async () => {