Skip to content

Commit c3a04cd

Browse files
d3r3kkd3r3kk
authored
Correct testlaunch python script to discover individual unit tests (microsoft#2332)
Correct testlaunch python script - Add tests! - reduce skipped-test noise in our CI
1 parent 065b377 commit c3a04cd

13 files changed

Lines changed: 363 additions & 190 deletions

File tree

‎news/2 Fixes/2241.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix `visualstudio_py_testLauncher` to stop breaking out of test discovery too soon.

‎pythonFiles/PythonTools/visualstudio_py_testlauncher.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,6 @@ def main():
297297
testId = m.id()
298298
if testId.startswith(opts.tests[0]):
299299
suite = cls
300-
break
301300
if testId == opts.tests[0]:
302301
tests = m
303302
break

‎src/test/common.ts‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@ import * as path from 'path';
33
import { ConfigurationTarget, Uri, workspace } from 'vscode';
44
import { PythonSettings } from '../client/common/configSettings';
55
import { EXTENSION_ROOT_DIR } from '../client/common/constants';
6-
import { sleep } from './core';
6+
import { sleep } from '../client/common/core.utils';
77
import { IS_MULTI_ROOT_TEST } from './initialize';
8+
export { sleep } from './core';
89

9-
export * from './core';
10-
11-
// tslint:disable:no-non-null-assertion no-unsafe-any await-promise no-any no-use-before-declare no-string-based-set-timeout no-unsafe-any no-any no-invalid-this
10+
// tslint:disable:no-invalid-this no-any
1211

1312
const fileInNonRootWorkspace = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'dummy.py');
1413
export const rootWorkspaceUri = getWorkspaceRoot();
@@ -35,10 +34,26 @@ export async function updateSetting(setting: PythonSettingKeys, value: {} | unde
3534
return;
3635
}
3736
await settings.update(setting, value, configTarget);
37+
38+
// We've experienced trouble with .update in the past, where VSC returns stale data even
39+
// after invoking the update method. This issue has regressed a few times as well. This
40+
// delay is merely a backup to ensure it extension doesn't break the tests due to similar
41+
// regressions in VSC:
3842
await sleep(2000);
43+
// ... please see issue #2356 and PR #2332 for a discussion on the matter
44+
3945
PythonSettings.dispose();
4046
}
4147

48+
// In some tests we will be mocking VS Code API (mocked classes)
49+
const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python';
50+
51+
export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder);
52+
53+
export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath);
54+
55+
export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)();
56+
4257
function getWorkspaceRoot() {
4358
if (!Array.isArray(workspace.workspaceFolders) || workspace.workspaceFolders.length === 0) {
4459
return Uri.file(path.join(EXTENSION_ROOT_DIR, 'src', 'test'));
@@ -108,12 +123,6 @@ export async function deleteFile(file: string) {
108123
}
109124
}
110125

111-
// In some tests we will be mocking VS Code API (mocked classes)
112-
const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python';
113-
export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder);
114-
export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath);
115-
export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)();
116-
117126
function getPythonPath(): string {
118127
if (process.env.CI_PYTHON_PATH && fs.existsSync(process.env.CI_PYTHON_PATH)) {
119128
return process.env.CI_PYTHON_PATH;

‎src/test/common/installer/installer.unit.test.ts‎

Lines changed: 37 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -112,47 +112,43 @@ suite('Module Installer', () => {
112112
moduleInstaller.verify(m => m.installModule(TypeMoq.It.isValue(moduleName), TypeMoq.It.isValue(resource)), TypeMoq.Times.once());
113113
}
114114
});
115-
test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () {
116-
if (product.value === Product.unittest) {
117-
return this.skip();
118-
}
119-
workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
120-
.returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
121-
.verifiable(TypeMoq.Times.exactly(resource ? 5 : 0));
122-
app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
123-
.returns(() => promptDeferred.promise)
124-
.verifiable(TypeMoq.Times.once());
125-
126-
// Display first prompt.
127-
installer.promptToInstall(product.value, resource).ignoreErrors();
128-
129-
// Display a few more prompts.
130-
installer.promptToInstall(product.value, resource).ignoreErrors();
131-
installer.promptToInstall(product.value, resource).ignoreErrors();
132-
installer.promptToInstall(product.value, resource).ignoreErrors();
133-
installer.promptToInstall(product.value, resource).ignoreErrors();
134-
135-
app.verifyAll();
136-
workspaceService.verifyAll();
137-
});
138-
test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () {
139-
if (product.value === Product.unittest) {
140-
return this.skip();
141-
}
142-
workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
143-
.returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
144-
.verifiable(TypeMoq.Times.exactly(resource ? 3 : 0));
145-
app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
146-
.returns(() => Promise.resolve(undefined))
147-
.verifiable(TypeMoq.Times.exactly(3));
148-
149-
await installer.promptToInstall(product.value, resource);
150-
await installer.promptToInstall(product.value, resource);
151-
await installer.promptToInstall(product.value, resource);
152-
153-
app.verifyAll();
154-
workspaceService.verifyAll();
155-
});
115+
if (product.value !== Product.unittest) {
116+
test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => {
117+
workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
118+
.returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
119+
.verifiable(TypeMoq.Times.exactly(resource ? 5 : 0));
120+
app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
121+
.returns(() => promptDeferred.promise)
122+
.verifiable(TypeMoq.Times.once());
123+
124+
// Display first prompt.
125+
installer.promptToInstall(product.value, resource).ignoreErrors();
126+
127+
// Display a few more prompts.
128+
installer.promptToInstall(product.value, resource).ignoreErrors();
129+
installer.promptToInstall(product.value, resource).ignoreErrors();
130+
installer.promptToInstall(product.value, resource).ignoreErrors();
131+
installer.promptToInstall(product.value, resource).ignoreErrors();
132+
133+
app.verifyAll();
134+
workspaceService.verifyAll();
135+
});
136+
test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => {
137+
workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!)))
138+
.returns(() => TypeMoq.Mock.ofType<WorkspaceFolder>().object)
139+
.verifiable(TypeMoq.Times.exactly(resource ? 3 : 0));
140+
app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
141+
.returns(() => Promise.resolve(undefined))
142+
.verifiable(TypeMoq.Times.exactly(3));
143+
144+
await installer.promptToInstall(product.value, resource);
145+
await installer.promptToInstall(product.value, resource);
146+
await installer.promptToInstall(product.value, resource);
147+
148+
app.verifyAll();
149+
workspaceService.verifyAll();
150+
});
151+
}
156152
}
157153
}
158154
});

‎src/test/pythonFiles/testFiles/multi/tests/more_tests/__init__.py‎

Whitespace-only changes.

‎src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import sys
2+
import os
3+
4+
import unittest
5+
6+
class Test_test3(unittest.TestCase):
7+
def test_3A(self):
8+
self.assertEqual(1, 2-1, "Not implemented")
9+
10+
def test_3B(self):
11+
self.assertEqual(1, 1, 'Not equal')
12+
13+
@unittest.skip("demonstrating skipping")
14+
def test_3C(self):
15+
self.assertEqual(1, 1, 'Not equal')
16+
17+
18+
if __name__ == '__main__':
19+
unittest.main()

‎src/test/pythonFiles/testFiles/multi/tests/test_one.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import sys
2+
import os
3+
4+
import unittest
5+
6+
class Test_test1(unittest.TestCase):
7+
def test_A(self):
8+
self.fail("Not implemented")
9+
10+
def test_B(self):
11+
self.assertEqual(1, 1, 'Not equal')
12+
13+
@unittest.skip("demonstrating skipping")
14+
def test_c(self):
15+
self.assertEqual(1, 1, 'Not equal')
16+
17+
18+
if __name__ == '__main__':
19+
unittest.main()

‎src/test/pythonFiles/testFiles/multi/tests/test_two.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import sys
2+
import os
3+
4+
import unittest
5+
6+
class Test_test2(unittest.TestCase):
7+
def test_2A(self):
8+
self.fail("Not implemented")
9+
10+
def test_2B(self):
11+
self.assertEqual(1, 1, 'Not equal')
12+
13+
@unittest.skip("demonstrating skipping")
14+
def test_2C(self):
15+
self.assertEqual(1, 1, 'Not equal')
16+
17+
18+
if __name__ == '__main__':
19+
unittest.main()

0 commit comments

Comments
 (0)