microsoft · d3r3kk · Aug 8, 2018 · Aug 3, 2018 · Aug 8, 2018 · Aug 8, 2018
diff --git a/news/2 Fixes/2241.md b/news/2 Fixes/2241.md
@@ -0,0 +1 @@
+Fix `visualstudio_py_testLauncher` to stop breaking out of test discovery too soon.
diff --git a/pythonFiles/PythonTools/visualstudio_py_testlauncher.py b/pythonFiles/PythonTools/visualstudio_py_testlauncher.py
@@ -297,7 +297,6 @@ def main():
                                 testId = m.id()
                                 if testId.startswith(opts.tests[0]):
                                     suite = cls
-                                    break
                                 if testId == opts.tests[0]:
                                     tests = m
                                     break

diff --git a/src/test/common.ts b/src/test/common.ts
@@ -3,12 +3,11 @@ import * as path from 'path';
 import { ConfigurationTarget, Uri, workspace } from 'vscode';
 import { PythonSettings } from '../client/common/configSettings';
 import { EXTENSION_ROOT_DIR } from '../client/common/constants';
-import { sleep } from './core';
+import { sleep } from '../client/common/core.utils';
 import { IS_MULTI_ROOT_TEST } from './initialize';
+export { sleep } from './core';
 
-export * from './core';
-
-// tslint:disable:no-non-null-assertion no-unsafe-any await-promise no-any no-use-before-declare no-string-based-set-timeout no-unsafe-any no-any no-invalid-this
+// tslint:disable:no-invalid-this no-any
 
 const fileInNonRootWorkspace = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'dummy.py');
 export const rootWorkspaceUri = getWorkspaceRoot();
@@ -35,10 +34,26 @@ export async function updateSetting(setting: PythonSettingKeys, value: {} | unde
         return;
     }
     await settings.update(setting, value, configTarget);
+
+    // We've experienced trouble with .update in the past, where VSC returns stale data even
+    // after invoking the update method. This issue has regressed a few times as well. This
+    // delay is merely a backup to ensure it extension doesn't break the tests due to similar
+    // regressions in VSC:
     await sleep(2000);
+    // ... please see issue #2356 and PR #2332 for a discussion on the matter
+
     PythonSettings.dispose();
 }
 
+// In some tests we will be mocking VS Code API (mocked classes)
+const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python';
+
+export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder);
+
+export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath);
+
+export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)();
+
 function getWorkspaceRoot() {
     if (!Array.isArray(workspace.workspaceFolders) || workspace.workspaceFolders.length === 0) {
         return Uri.file(path.join(EXTENSION_ROOT_DIR, 'src', 'test'));
@@ -108,12 +123,6 @@ export async function deleteFile(file: string) {
     }
 }
 
-// In some tests we will be mocking VS Code API (mocked classes)
-const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python';
-export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder);
-export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath);
-export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)();
-
 function getPythonPath(): string {
     if (process.env.CI_PYTHON_PATH && fs.existsSync(process.env.CI_PYTHON_PATH)) {
         return process.env.CI_PYTHON_PATH;

diff --git a/src/test/common/installer/installer.unit.test.ts b/src/test/common/installer/installer.unit.test.ts
@@ -112,47 +112,43 @@ suite('Module Installer', () => {
                             moduleInstaller.verify(m => m.installModule(TypeMoq.It.isValue(moduleName), TypeMoq.It.isValue(resource)), TypeMoq.Times.once());
                         }
                     });
-                    test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () {
-                        if (product.value === Product.unittest) {
-                            return this.skip();
-                        }
-                        workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
-                            .returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
-                            .verifiable(TypeMoq.Times.exactly(resource ? 5 : 0));
-                        app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
-                            .returns(() => promptDeferred.promise)
-                            .verifiable(TypeMoq.Times.once());
-
-                        // Display first prompt.
-                        installer.promptToInstall(product.value, resource).ignoreErrors();
-
-                        // Display a few more prompts.
-                        installer.promptToInstall(product.value, resource).ignoreErrors();
-                        installer.promptToInstall(product.value, resource).ignoreErrors();
-                        installer.promptToInstall(product.value, resource).ignoreErrors();
-                        installer.promptToInstall(product.value, resource).ignoreErrors();
-
-                        app.verifyAll();
-                        workspaceService.verifyAll();
-                    });
-                    test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () {
-                        if (product.value === Product.unittest) {
-                            return this.skip();
-                        }
-                        workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
-                            .returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
-                            .verifiable(TypeMoq.Times.exactly(resource ? 3 : 0));
-                        app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
-                            .returns(() => Promise.resolve(undefined))
-                            .verifiable(TypeMoq.Times.exactly(3));
-
-                        await installer.promptToInstall(product.value, resource);
-                        await installer.promptToInstall(product.value, resource);
-                        await installer.promptToInstall(product.value, resource);
-
-                        app.verifyAll();
-                        workspaceService.verifyAll();
-                    });
+                    if (product.value !== Product.unittest) {
+                        test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => {
+                            workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
+                                .returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
+                                .verifiable(TypeMoq.Times.exactly(resource ? 5 : 0));
+                            app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
+                                .returns(() => promptDeferred.promise)
+                                .verifiable(TypeMoq.Times.once());
+
+                            // Display first prompt.
+                            installer.promptToInstall(product.value, resource).ignoreErrors();
+
+                            // Display a few more prompts.
+                            installer.promptToInstall(product.value, resource).ignoreErrors();
+                            installer.promptToInstall(product.value, resource).ignoreErrors();
+                            installer.promptToInstall(product.value, resource).ignoreErrors();
+                            installer.promptToInstall(product.value, resource).ignoreErrors();
+
+                            app.verifyAll();
+                            workspaceService.verifyAll();
+                        });
+                        test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => {
+                            workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
+                                .returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
+                                .verifiable(TypeMoq.Times.exactly(resource ? 3 : 0));
+                            app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
+                                .returns(() => Promise.resolve(undefined))
+                                .verifiable(TypeMoq.Times.exactly(3));
+
+                            await installer.promptToInstall(product.value, resource);
+                            await installer.promptToInstall(product.value, resource);
+                            await installer.promptToInstall(product.value, resource);
+
+                            app.verifyAll();
+                            workspaceService.verifyAll();
+                        });
+                    }
                 }
             }
         });

diff --git a/src/test/pythonFiles/testFiles/multi/tests/more_tests/__init__.py b/src/test/pythonFiles/testFiles/multi/tests/more_tests/__init__.py
diff --git a/src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py b/src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py
@@ -0,0 +1,19 @@
+import sys
+import os
+
+import unittest
+
+class Test_test3(unittest.TestCase):
+    def test_3A(self):
+        self.assertEqual(1, 2-1, "Not implemented")
+
+    def test_3B(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+    @unittest.skip("demonstrating skipping")
+    def test_3C(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/test/pythonFiles/testFiles/multi/tests/test_one.py b/src/test/pythonFiles/testFiles/multi/tests/test_one.py
@@ -0,0 +1,19 @@
+import sys
+import os
+
+import unittest
+
+class Test_test1(unittest.TestCase):
+    def test_A(self):
+        self.fail("Not implemented")
+
+    def test_B(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+    @unittest.skip("demonstrating skipping")
+    def test_c(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/test/pythonFiles/testFiles/multi/tests/test_two.py b/src/test/pythonFiles/testFiles/multi/tests/test_two.py
@@ -0,0 +1,19 @@
+import sys
+import os
+
+import unittest
+
+class Test_test2(unittest.TestCase):
+    def test_2A(self):
+        self.fail("Not implemented")
+
+    def test_2B(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+    @unittest.skip("demonstrating skipping")
+    def test_2C(self):
+        self.assertEqual(1, 1, 'Not equal')
+
+
+if __name__ == '__main__':
+    unittest.main()
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix `visualstudio_py_testLauncher` to stop breaking out of test discovery too soon.