👷 Change CI test sequence (run debugger tests first and separately) (microsoft#861)

DonJayamanne · web-flow · commit fe1a780f5e9a · 2018-02-22T10:06:01.000-08:00
Fixes microsoft#860 Also fixes a test when running Yapf tests on travis
diff --git a/.travis.yml b/.travis.yml
@@ -29,6 +29,13 @@ script:
   - yarn run clean
   - yarn run vscode:prepublish
   - yarn run cover:enable
+  - yarn run testDebugger --silent
+  - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
+        bash <(curl -s https://codecov.io/bash);
+    fi
+  - yarn run clean
+  - yarn run vscode:prepublish
+  - yarn run cover:enable
   - yarn run testSingleWorkspace --silent
   - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
         bash <(curl -s https://codecov.io/bash);
diff --git a/package.json b/package.json
@@ -1755,6 +1755,7 @@
         "compile": "tsc -watch -p ./",
         "postinstall": "node ./node_modules/vscode/bin/install",
         "test": "node ./out/test/standardTest.js && node ./out/test/multiRootTest.js",
+        "testDebugger": "node ./out/test/debuggerTest.js",
         "testSingleWorkspace": "node ./out/test/standardTest.js",
         "testMultiWorkspace": "node ./out/test/multiRootTest.js",
         "precommit": "node gulpfile.js",
diff --git a/src/client/common/net/socket/socketServer.ts b/src/client/common/net/socket/socketServer.ts
@@ -56,6 +56,9 @@ export class SocketServer extends EventEmitter implements ISocketServer {
         client.on('data', (data: Buffer) => {
             this.emit('data', client, data);
         });
+        client.on('error', (err: Error) => {
+            this.emit('error', client, err);
+        });
 
         client.on('timeout', d => {
             // let msg = "Debugger client timedout, " + d;
diff --git a/src/client/debugger/DebugClients/LocalDebugClient.ts b/src/client/debugger/DebugClients/LocalDebugClient.ts
@@ -66,22 +66,8 @@ export class LocalDebugClient extends DebugClient<LaunchRequestArguments> {
             this.debugServer!.Stop();
             this.debugServer = undefined;
         }
-        if (this.args.type === 'pythonExperimental' && this.pyProc) {
-            this.pyProc.kill();
-        }
         if (this.pyProc) {
-            try {
-                this.pyProc!.send('EXIT');
-                // tslint:disable-next-line:no-empty
-            } catch { }
-            try {
-                this.pyProc!.stdin.write('EXIT');
-                // tslint:disable-next-line:no-empty
-            } catch { }
-            try {
-                this.pyProc!.disconnect();
-                // tslint:disable-next-line:no-empty
-            } catch { }
+            this.pyProc.kill();
             this.pyProc = undefined;
         }
     }
diff --git a/src/client/debugger/Main.ts b/src/client/debugger/Main.ts
@@ -735,6 +735,10 @@ export class PythonDebugger extends LoggingDebugSession {
     }
 }
 
+process.stdin.on('error', () => { });
+process.stdout.on('error', () => { });
+process.stderr.on('error', () => { });
+
 process.on('uncaughtException', (err: Error) => {
     logger.error(`Uncaught Exception: ${err && err.message ? err.message : ''}`);
     logger.error(err && err.name ? err.name : '');
diff --git a/src/client/debugger/PythonProcess.ts b/src/client/debugger/PythonProcess.ts
@@ -398,9 +398,11 @@ export class PythonProcess extends EventEmitter implements IPythonProcess {
         proc.stderr.on('data', (error: string) => {
             this.emit("error", error.toString());
         });
+        proc.stderr.on('error', () => { });
         proc.stdout.on('data', (d: string) => {
             this.emit("output", undefined, d);
         });
+        proc.stdout.on('error', () => { });
         proc.on('close', () => {
             this.emit('detach');
         });
diff --git a/src/test/common/misc.test.ts b/src/test/common/misc.test.ts
@@ -8,7 +8,7 @@ import { isTestExecution } from '../../client/common/constants';
 
 // Defines a Mocha test suite to group tests of similar kind together
 suite('Common - Misc', () => {
-    test('Ensure its identified that we\re running unit tests', () => {
+    test('Ensure its identified that we\'re running unit tests', () => {
         expect(isTestExecution()).to.be.equal(true, 'incorrect');
     });
 });
diff --git a/src/test/constants.ts b/src/test/constants.ts
@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable:no-string-literal
+import { workspace } from 'vscode';
+
+export const IS_CI_SERVER = (typeof process.env['TRAVIS'] === 'string' ? process.env['TRAVIS'] : '') === 'true';
+export const TEST_TIMEOUT = 25000;
+export const IS_MULTI_ROOT_TEST = isMultitrootTest();
+export const IS_CI_SERVER_TEST_DEBUGGER = process.env['IS_CI_SERVER_TEST_DEBUGGER'] === '1';
+// If running on CI server, then run debugger tests ONLY if the corresponding flag is enabled.
+export const TEST_DEBUGGER = IS_CI_SERVER ? IS_CI_SERVER_TEST_DEBUGGER : true;
+
+function isMultitrootTest() {
+    return Array.isArray(workspace.workspaceFolders) && workspace.workspaceFolders.length > 1;
+}
diff --git a/src/test/debugger/misc.test.ts b/src/test/debugger/misc.test.ts
@@ -11,7 +11,7 @@ import { DebugClient } from 'vscode-debugadapter-testsupport';
 import { DebugProtocol } from 'vscode-debugprotocol';
 import { LaunchRequestArguments } from '../../client/debugger/Common/Contracts';
 import { sleep } from '../common';
-import { IS_CI_SERVER, IS_MULTI_ROOT_TEST } from '../initialize';
+import { IS_CI_SERVER, IS_MULTI_ROOT_TEST, TEST_DEBUGGER } from '../initialize';
 
 const isProcessRunning = require('is-running') as (number) => boolean;
 
@@ -30,7 +30,7 @@ const EXPERIMENTAL_DEBUG_ADAPTER = path.join(__dirname, '..', '..', 'client', 'd
 
         let debugClient: DebugClient;
         setup(async function () {
-            if (!IS_MULTI_ROOT_TEST) {
+            if (!IS_MULTI_ROOT_TEST || !TEST_DEBUGGER) {
                 this.skip();
             }
             // Temporary, untill new version of PTVSD is bundled we cannot run tests
@@ -134,7 +134,7 @@ const EXPERIMENTAL_DEBUG_ADAPTER = path.join(__dirname, '..', '..', 'client', 'd
                 debugClient.waitForEvent('terminated')
             ]);
         });
-        test('Ensure threadid is int32', async () => {
+        test('Ensure threadid is int32', async function () {
             if (debuggerType !== 'python') {
                 return this.skip();
             }
diff --git a/src/test/debugger/portAndHost.test.ts b/src/test/debugger/portAndHost.test.ts
@@ -8,7 +8,7 @@ import * as net from 'net';
 import * as path from 'path';
 import { DebugClient } from 'vscode-debugadapter-testsupport';
 import { LaunchRequestArguments } from '../../client/debugger/Common/Contracts';
-import { IS_MULTI_ROOT_TEST } from '../initialize';
+import { IS_MULTI_ROOT_TEST, TEST_DEBUGGER } from '../initialize';
 
 use(chaiAsPromised);
 
@@ -24,7 +24,7 @@ const EXPERIMENTAL_DEBUG_ADAPTER = path.join(__dirname, '..', '..', 'client', 'd
     suite(`Standard Debugging of ports and hosts: ${debuggerType}`, () => {
         let debugClient: DebugClient;
         setup(async function () {
-            if (!IS_MULTI_ROOT_TEST) {
+            if (!IS_MULTI_ROOT_TEST || !TEST_DEBUGGER) {
                 // tslint:disable-next-line:no-invalid-this
                 this.skip();
             }
diff --git a/src/test/debuggerTest.ts b/src/test/debuggerTest.ts
@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable:no-console no-require-imports no-var-requires
+
+import * as path from 'path';
+
+process.env.CODE_TESTS_WORKSPACE = path.join(__dirname, '..', '..', 'src', 'testMultiRootWkspc', 'multi.code-workspace');
+process.env.IS_CI_SERVER_TEST_DEBUGGER = '1';
+
+function start() {
+    console.log('*'.repeat(100));
+    console.log('Start Debugger tests');
+    require('../../node_modules/vscode/bin/test');
+}
+start();
diff --git a/src/test/index.ts b/src/test/index.ts
@@ -3,22 +3,27 @@ if ((Reflect as any).metadata === undefined) {
     // tslint:disable-next-line:no-require-imports no-var-requires
     require('reflect-metadata');
 }
-import { workspace } from 'vscode';
 import { MochaSetupOptions } from 'vscode/lib/testrunner';
+import { IS_CI_SERVER, IS_CI_SERVER_TEST_DEBUGGER, IS_MULTI_ROOT_TEST } from './constants';
 import * as testRunner from './testRunner';
 
 process.env.VSC_PYTHON_CI_TEST = '1';
-process.env.IS_MULTI_ROOT_TEST = (Array.isArray(workspace.workspaceFolders) && workspace.workspaceFolders.length > 1);
+process.env.IS_MULTI_ROOT_TEST = IS_MULTI_ROOT_TEST;
+
+// If running on CI server and we're running the debugger tests, then ensure we only run debug tests.
+// We do this to ensure we only run debugger test, as debugger tests are very flaky on CI.
+// So the solution is to run them separately and first on CI.
+const grep = IS_CI_SERVER && IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : undefined;
 
 // You can directly control Mocha options by uncommenting the following lines.
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info.
 // Hack, as retries is not supported as setting in tsd.
-// tslint:disable-next-line:no-any
 const options: MochaSetupOptions & { retries: number } = {
     ui: 'tdd',
     useColors: true,
     timeout: 25000,
-    retries: 3
+    retries: 3,
+    grep
 };
 testRunner.configure(options, { coverageConfig: '../coverconfig.json' });
 module.exports = testRunner;
diff --git a/src/test/initialize.ts b/src/test/initialize.ts
@@ -1,23 +1,22 @@
+// tslint:disable:no-string-literal
+
 import * as fs from 'fs';
 import * as path from 'path';
 import * as vscode from 'vscode';
 import { PythonSettings } from '../client/common/configSettings';
 import { activated } from '../client/extension';
 import { clearPythonPathInWorkspaceFolder, resetGlobalPythonPathSetting, setPythonPathInWorkspaceRoot } from './common';
 
+export * from './constants';
+
 const dummyPythonFile = path.join(__dirname, '..', '..', 'src', 'test', 'pythonFiles', 'dummy.py');
 const multirootPath = path.join(__dirname, '..', '..', 'src', 'testMultiRootWkspc');
 const workspace3Uri = vscode.Uri.file(path.join(multirootPath, 'workspace3'));
 
 //First thing to be executed.
-// tslint:disable-next-line:no-string-literal
 process.env['VSC_PYTHON_CI_TEST'] = '1';
 
 const PYTHON_PATH = getPythonPath();
-// tslint:disable-next-line:no-string-literal prefer-template
-export const IS_CI_SERVER = (typeof process.env['TRAVIS'] === 'string' ? process.env['TRAVIS'] : '') === 'true';
-export const TEST_TIMEOUT = 25000;
-export const IS_MULTI_ROOT_TEST = isMultitrootTest();
 
 // Ability to use custom python environments for testing
 export async function initializePython() {
@@ -65,7 +64,3 @@ function getPythonPath(): string {
     }
     return 'python';
 }
-
-function isMultitrootTest() {
-    return Array.isArray(vscode.workspace.workspaceFolders) && vscode.workspace.workspaceFolders.length > 1;
-}
diff --git a/src/test/multiRootTest.ts b/src/test/multiRootTest.ts
@@ -1,10 +1,13 @@
+// tslint:disable:no-console no-require-imports no-var-requires
+
 import * as path from 'path';
 
 process.env.CODE_TESTS_WORKSPACE = path.join(__dirname, '..', '..', 'src', 'testMultiRootWkspc', 'multi.code-workspace');
+process.env.IS_CI_SERVER_TEST_DEBUGGER = '';
 
 function start() {
-    // tslint:disable-next-line:no-console
-    console.log('start Multiroot tests');
+    console.log('*'.repeat(100));
+    console.log('Start Multiroot tests');
     require('../../node_modules/vscode/bin/test');
 }
 start();
diff --git a/src/test/pythonFiles/formatting/dummy.ts b/src/test/pythonFiles/formatting/dummy.ts
@@ -0,0 +1,4 @@
+// Dummy ts file to ensure this folder gets created in output directory.
+
+// Code to ensure linter doesn't complain about empty files.
+const a = '1';
diff --git a/src/test/standardTest.ts b/src/test/standardTest.ts
@@ -1,8 +1,13 @@
+// tslint:disable:no-console no-require-imports no-var-requires
+
 import * as path from 'path';
 
 process.env.CODE_TESTS_WORKSPACE = path.join(__dirname, '..', '..', 'src', 'test');
+process.env.IS_CI_SERVER_TEST_DEBUGGER = '';
 
 function start() {
+    console.log('*'.repeat(100));
+    console.log('Start Standard tests');
     require('../../node_modules/vscode/bin/test');
 }
 start();

Original file line number	Diff line number	Diff line change
`@@ -735,6 +735,10 @@ export class PythonDebugger extends LoggingDebugSession {`
`735`	`735`	`}`
`736`	`736`	`}`
`737`	`737`
	`738`	`+process.stdin.on('error', () => { });`
	`739`	`+process.stdout.on('error', () => { });`
	`740`	`+process.stderr.on('error', () => { });`
	`741`	`+`
`738`	`742`	`process.on('uncaughtException', (err: Error) => {`
`739`	`743`	logger.error(`Uncaught Exception: ${err && err.message ? err.message : ''}`);
`740`	`744`	`logger.error(err && err.name ? err.name : '');`