diff --git a/news/2 Fixes/2241.md b/news/2 Fixes/2241.md new file mode 100644 index 000000000000..a00f6c17c249 --- /dev/null +++ b/news/2 Fixes/2241.md @@ -0,0 +1 @@ +Fix `visualstudio_py_testLauncher` to stop breaking out of test discovery too soon. \ No newline at end of file diff --git a/pythonFiles/PythonTools/visualstudio_py_testlauncher.py b/pythonFiles/PythonTools/visualstudio_py_testlauncher.py index 270d87be3399..f68cc2ee5a88 100644 --- a/pythonFiles/PythonTools/visualstudio_py_testlauncher.py +++ b/pythonFiles/PythonTools/visualstudio_py_testlauncher.py @@ -297,7 +297,6 @@ def main(): testId = m.id() if testId.startswith(opts.tests[0]): suite = cls - break if testId == opts.tests[0]: tests = m break diff --git a/src/test/common.ts b/src/test/common.ts index d2645fd5e31c..cf1b6dfbd732 100644 --- a/src/test/common.ts +++ b/src/test/common.ts @@ -3,12 +3,11 @@ import * as path from 'path'; import { ConfigurationTarget, Uri, workspace } from 'vscode'; import { PythonSettings } from '../client/common/configSettings'; import { EXTENSION_ROOT_DIR } from '../client/common/constants'; -import { sleep } from './core'; +import { sleep } from '../client/common/core.utils'; import { IS_MULTI_ROOT_TEST } from './initialize'; +export { sleep } from './core'; -export * from './core'; - -// tslint:disable:no-non-null-assertion no-unsafe-any await-promise no-any no-use-before-declare no-string-based-set-timeout no-unsafe-any no-any no-invalid-this +// tslint:disable:no-invalid-this no-any const fileInNonRootWorkspace = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'dummy.py'); export const rootWorkspaceUri = getWorkspaceRoot(); @@ -35,10 +34,26 @@ export async function updateSetting(setting: PythonSettingKeys, value: {} | unde return; } await settings.update(setting, value, configTarget); + + // We've experienced trouble with .update in the past, where VSC returns stale data even + // after invoking the update method. This issue has regressed a few times as well. This + // delay is merely a backup to ensure it extension doesn't break the tests due to similar + // regressions in VSC: await sleep(2000); + // ... please see issue #2356 and PR #2332 for a discussion on the matter + PythonSettings.dispose(); } +// In some tests we will be mocking VS Code API (mocked classes) +const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python'; + +export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder); + +export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath); + +export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)(); + function getWorkspaceRoot() { if (!Array.isArray(workspace.workspaceFolders) || workspace.workspaceFolders.length === 0) { return Uri.file(path.join(EXTENSION_ROOT_DIR, 'src', 'test')); @@ -108,12 +123,6 @@ export async function deleteFile(file: string) { } } -// In some tests we will be mocking VS Code API (mocked classes) -const globalPythonPathSetting = workspace.getConfiguration('python') ? workspace.getConfiguration('python').inspect('pythonPath')!.globalValue : 'python'; -export const clearPythonPathInWorkspaceFolder = async (resource: string | Uri) => retryAsync(setPythonPathInWorkspace)(resource, ConfigurationTarget.WorkspaceFolder); -export const setPythonPathInWorkspaceRoot = async (pythonPath: string) => retryAsync(setPythonPathInWorkspace)(undefined, ConfigurationTarget.Workspace, pythonPath); -export const resetGlobalPythonPathSetting = async () => retryAsync(restoreGlobalPythonPathSetting)(); - function getPythonPath(): string { if (process.env.CI_PYTHON_PATH && fs.existsSync(process.env.CI_PYTHON_PATH)) { return process.env.CI_PYTHON_PATH; diff --git a/src/test/common/installer/installer.unit.test.ts b/src/test/common/installer/installer.unit.test.ts index 09c4de523d1d..a07ca2e34e1a 100644 --- a/src/test/common/installer/installer.unit.test.ts +++ b/src/test/common/installer/installer.unit.test.ts @@ -112,47 +112,43 @@ suite('Module Installer', () => { moduleInstaller.verify(m => m.installModule(TypeMoq.It.isValue(moduleName), TypeMoq.It.isValue(resource)), TypeMoq.Times.once()); } }); - test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () { - if (product.value === Product.unittest) { - return this.skip(); - } - workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!))) - .returns(() => TypeMoq.Mock.ofType().object) - .verifiable(TypeMoq.Times.exactly(resource ? 5 : 0)); - app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) - .returns(() => promptDeferred.promise) - .verifiable(TypeMoq.Times.once()); - - // Display first prompt. - installer.promptToInstall(product.value, resource).ignoreErrors(); - - // Display a few more prompts. - installer.promptToInstall(product.value, resource).ignoreErrors(); - installer.promptToInstall(product.value, resource).ignoreErrors(); - installer.promptToInstall(product.value, resource).ignoreErrors(); - installer.promptToInstall(product.value, resource).ignoreErrors(); - - app.verifyAll(); - workspaceService.verifyAll(); - }); - test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async function () { - if (product.value === Product.unittest) { - return this.skip(); - } - workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!))) - .returns(() => TypeMoq.Mock.ofType().object) - .verifiable(TypeMoq.Times.exactly(resource ? 3 : 0)); - app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) - .returns(() => Promise.resolve(undefined)) - .verifiable(TypeMoq.Times.exactly(3)); - - await installer.promptToInstall(product.value, resource); - await installer.promptToInstall(product.value, resource); - await installer.promptToInstall(product.value, resource); - - app.verifyAll(); - workspaceService.verifyAll(); - }); + if (product.value !== Product.unittest) { + test(`Ensure the prompt is displayed only once, untill the prompt is closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => { + workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!))) + .returns(() => TypeMoq.Mock.ofType().object) + .verifiable(TypeMoq.Times.exactly(resource ? 5 : 0)); + app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns(() => promptDeferred.promise) + .verifiable(TypeMoq.Times.once()); + + // Display first prompt. + installer.promptToInstall(product.value, resource).ignoreErrors(); + + // Display a few more prompts. + installer.promptToInstall(product.value, resource).ignoreErrors(); + installer.promptToInstall(product.value, resource).ignoreErrors(); + installer.promptToInstall(product.value, resource).ignoreErrors(); + installer.promptToInstall(product.value, resource).ignoreErrors(); + + app.verifyAll(); + workspaceService.verifyAll(); + }); + test(`Ensure the prompt is displayed again when previous prompt has been closed, ${product.name} (${resource ? 'With a resource' : 'without a resource'})`, async () => { + workspaceService.setup(w => w.getWorkspaceFolder(TypeMoq.It.isValue(resource!))) + .returns(() => TypeMoq.Mock.ofType().object) + .verifiable(TypeMoq.Times.exactly(resource ? 3 : 0)); + app.setup(a => a.showErrorMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns(() => Promise.resolve(undefined)) + .verifiable(TypeMoq.Times.exactly(3)); + + await installer.promptToInstall(product.value, resource); + await installer.promptToInstall(product.value, resource); + await installer.promptToInstall(product.value, resource); + + app.verifyAll(); + workspaceService.verifyAll(); + }); + } } } }); diff --git a/src/test/pythonFiles/testFiles/multi/tests/more_tests/__init__.py b/src/test/pythonFiles/testFiles/multi/tests/more_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py b/src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py new file mode 100644 index 000000000000..9cea70ae7ca6 --- /dev/null +++ b/src/test/pythonFiles/testFiles/multi/tests/more_tests/test_three.py @@ -0,0 +1,19 @@ +import sys +import os + +import unittest + +class Test_test3(unittest.TestCase): + def test_3A(self): + self.assertEqual(1, 2-1, "Not implemented") + + def test_3B(self): + self.assertEqual(1, 1, 'Not equal') + + @unittest.skip("demonstrating skipping") + def test_3C(self): + self.assertEqual(1, 1, 'Not equal') + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/pythonFiles/testFiles/multi/tests/test_one.py b/src/test/pythonFiles/testFiles/multi/tests/test_one.py new file mode 100644 index 000000000000..e869986b6ead --- /dev/null +++ b/src/test/pythonFiles/testFiles/multi/tests/test_one.py @@ -0,0 +1,19 @@ +import sys +import os + +import unittest + +class Test_test1(unittest.TestCase): + def test_A(self): + self.fail("Not implemented") + + def test_B(self): + self.assertEqual(1, 1, 'Not equal') + + @unittest.skip("demonstrating skipping") + def test_c(self): + self.assertEqual(1, 1, 'Not equal') + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/pythonFiles/testFiles/multi/tests/test_two.py b/src/test/pythonFiles/testFiles/multi/tests/test_two.py new file mode 100644 index 000000000000..f3fef9c9b1eb --- /dev/null +++ b/src/test/pythonFiles/testFiles/multi/tests/test_two.py @@ -0,0 +1,19 @@ +import sys +import os + +import unittest + +class Test_test2(unittest.TestCase): + def test_2A(self): + self.fail("Not implemented") + + def test_2B(self): + self.assertEqual(1, 1, 'Not equal') + + @unittest.skip("demonstrating skipping") + def test_2C(self): + self.assertEqual(1, 1, 'Not equal') + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/unittests/argsService.unit.test.ts b/src/test/unittests/argsService.unit.test.ts index e0ca1043c84e..8f880c236574 100644 --- a/src/test/unittests/argsService.unit.test.ts +++ b/src/test/unittests/argsService.unit.test.ts @@ -3,11 +3,11 @@ 'use strict'; -// tslint:disable:max-func-body-length no-any no-conditional-assignment no-increment-decrement no-invalid-this insecure-random +// tslint:disable:max-func-body-length + import { fail } from 'assert'; import { expect } from 'chai'; import { spawnSync } from 'child_process'; -import * as path from 'path'; import * as typeMoq from 'typemoq'; import { EnumEx } from '../../client/common/enumUtils'; import { ILogger, Product } from '../../client/common/types'; @@ -19,7 +19,7 @@ import { IArgumentsHelper, IArgumentsService } from '../../client/unittests/type import { ArgumentsService as UnitTestArgumentsService } from '../../client/unittests/unittest/services/argsService'; import { PYTHON_PATH } from '../common'; -suite('Unit Tests - argsService', () => { +suite('ArgsService: Common', () => { [Product.unittest, Product.nosetest, Product.pytest] .forEach(product => { const productNames = EnumEx.getNamesAndValues(Product); @@ -99,136 +99,6 @@ suite('Unit Tests - argsService', () => { expect(value).to.deep.equal(['abcd', 'xyz']); } }); - test('Test getting the test folder in unittest with -s', function () { - if (product !== Product.unittest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--three', '-s', dir]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in unittest with -s in the middle', function () { - if (product !== Product.unittest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--three', '-s', dir, 'some other', '--value', '1234']; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in unittest with --start-directory', function () { - if (product !== Product.unittest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--three', '--start-directory', dir]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in unittest with --start-directory in the middle', function () { - if (product !== Product.unittest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--three', '--start-directory', dir, 'some other', '--value', '1234']; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in nosetest', function () { - if (product !== Product.nosetest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--three', dir]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in nosetest (with multiple dirs)', function () { - if (product !== Product.nosetest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const dir2 = path.join('a', 'b', '2'); - const args = ['anzy', '--one', '--three', dir, dir2]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(2); - expect(testDirs[0]).to.equal(dir); - expect(testDirs[1]).to.equal(dir2); - }); - test('Test getting the test folder in pytest', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', '--rootdir', dir]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in pytest (with multiple dirs)', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const dir2 = path.join('a', 'b', '2'); - const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(2); - expect(testDirs[0]).to.equal(dir); - expect(testDirs[1]).to.equal(dir2); - }); - test('Test getting the test folder in pytest (with multiple dirs in the middle)', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const dir2 = path.join('a', 'b', '2'); - const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2, '-xyz']; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(2); - expect(testDirs[0]).to.equal(dir); - expect(testDirs[1]).to.equal(dir2); - }); - test('Test getting the test folder in pytest (with single positional dir)', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const args = ['anzy', '--one', dir]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(1); - expect(testDirs[0]).to.equal(dir); - }); - test('Test getting the test folder in pytest (with multiple positional dirs)', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const dir2 = path.join('a', 'b', '2'); - const args = ['anzy', '--one', dir, dir2]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(2); - expect(testDirs[0]).to.equal(dir); - expect(testDirs[1]).to.equal(dir2); - }); - test('Test getting the test folder in pytest (with multiple dirs excluding python files)', function () { - if (product !== Product.pytest) { - return this.skip(); - } - const dir = path.join('a', 'b', 'c'); - const dir2 = path.join('a', 'b', '2'); - const args = ['anzy', '--one', dir, dir2, path.join(dir, 'one.py')]; - const testDirs = argumentsService.getTestFolders(args); - expect(testDirs).to.be.lengthOf(2); - expect(testDirs[0]).to.equal(dir); - expect(testDirs[1]).to.equal(dir2); - }); test('Test filtering of arguments', () => { const args: string[] = []; const knownOptions = argumentsService.getKnownOptions(); @@ -255,7 +125,6 @@ suite('Unit Tests - argsService', () => { }); function getOptions(product: Product, moduleName: string, withValues: boolean) { - // const result = spawnSync('/Users/donjayamanne/Desktop/Development/PythonStuff/vscodePythonTesting/testingFolder/venv/bin/python', ['-m', moduleName, '-h']); const result = spawnSync(PYTHON_PATH, ['-m', moduleName, '-h']); const output = result.stdout.toString(); @@ -291,12 +160,13 @@ function getOptionsWithArguments(output: string) { function getMatches(pattern, str) { const matches: string[] = []; const regex = new RegExp(pattern, 'gm'); - let result; - while ((result = regex.exec(str)) !== null) { + let result: RegExpExecArray | null = regex.exec(str); + while (result !== null) { if (result.index === regex.lastIndex) { - regex.lastIndex++; + regex.lastIndex += 1; } matches.push(result[1].trim()); + result = regex.exec(str); } return matches .sort() diff --git a/src/test/unittests/nosetest/nosetest.argsService.unit.test.ts b/src/test/unittests/nosetest/nosetest.argsService.unit.test.ts new file mode 100644 index 000000000000..655834008b9f --- /dev/null +++ b/src/test/unittests/nosetest/nosetest.argsService.unit.test.ts @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { expect } from 'chai'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { ILogger } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { ArgumentsHelper } from '../../../client/unittests/common/argumentsHelper'; +import { ArgumentsService as NoseTestArgumentsService } from '../../../client/unittests/nosetest/services/argsService'; +import { IArgumentsHelper } from '../../../client/unittests/types'; + +suite('ArgsService: nosetest', () => { + let argumentsService: NoseTestArgumentsService; + + suiteSetup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + const logger = typeMoq.Mock.ofType(); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(ILogger), typeMoq.It.isAny())) + .returns(() => logger.object); + + const argsHelper = new ArgumentsHelper(serviceContainer.object); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(IArgumentsHelper), typeMoq.It.isAny())) + .returns(() => argsHelper); + + argumentsService = new NoseTestArgumentsService(serviceContainer.object); + }); + + test('Test getting the test folder in nosetest', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in nosetest (with multiple dirs)', () => { + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--three', dir, dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); +}); diff --git a/src/test/unittests/pytest/pytest.argsService.unit.test.ts b/src/test/unittests/pytest/pytest.argsService.unit.test.ts new file mode 100644 index 000000000000..bb50ecf8bac1 --- /dev/null +++ b/src/test/unittests/pytest/pytest.argsService.unit.test.ts @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { expect } from 'chai'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { ILogger } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { ArgumentsHelper } from '../../../client/unittests/common/argumentsHelper'; +import { ArgumentsService as PyTestArgumentsService } from '../../../client/unittests/pytest/services/argsService'; +import { IArgumentsHelper } from '../../../client/unittests/types'; + +suite('ArgsService: nosetest', () => { + let argumentsService: PyTestArgumentsService; + + suiteSetup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + const logger = typeMoq.Mock.ofType(); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(ILogger), typeMoq.It.isAny())) + .returns(() => logger.object); + + const argsHelper = new ArgumentsHelper(serviceContainer.object); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(IArgumentsHelper), typeMoq.It.isAny())) + .returns(() => argsHelper); + + argumentsService = new PyTestArgumentsService(serviceContainer.object); + }); + + test('Test getting the test folder in pytest', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--rootdir', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in pytest (with multiple dirs)', () => { + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with multiple dirs in the middle)', () => { + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2, '-xyz']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with single positional dir)', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in pytest (with multiple positional dirs)', () => { + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', dir, dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with multiple dirs excluding python files)', () => { + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', dir, dir2, path.join(dir, 'one.py')]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); +}); diff --git a/src/test/unittests/unittest/unittest.argsService.unit.test.ts b/src/test/unittests/unittest/unittest.argsService.unit.test.ts new file mode 100644 index 000000000000..3cccf3dbb0c2 --- /dev/null +++ b/src/test/unittests/unittest/unittest.argsService.unit.test.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { expect } from 'chai'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { ILogger } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { ArgumentsHelper } from '../../../client/unittests/common/argumentsHelper'; +import { IArgumentsHelper } from '../../../client/unittests/types'; +import { ArgumentsService as UnittestArgumentsService } from '../../../client/unittests/unittest/services/argsService'; + +suite('ArgsService: nosetest', () => { + let argumentsService: UnittestArgumentsService; + + suiteSetup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + const logger = typeMoq.Mock.ofType(); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(ILogger), typeMoq.It.isAny())) + .returns(() => logger.object); + + const argsHelper = new ArgumentsHelper(serviceContainer.object); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(IArgumentsHelper), typeMoq.It.isAny())) + .returns(() => argsHelper); + + argumentsService = new UnittestArgumentsService(serviceContainer.object); + }); + + test('Test getting the test folder in unittest with -s', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '-s', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with -s in the middle', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '-s', dir, 'some other', '--value', '1234']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with --start-directory', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '--start-directory', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with --start-directory in the middle', () => { + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '--start-directory', dir, 'some other', '--value', '1234']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); +}); diff --git a/src/test/unittests/unittest/unittest.test.ts b/src/test/unittests/unittest/unittest.test.ts index 4152ef540be9..6d8256d745e1 100644 --- a/src/test/unittests/unittest/unittest.test.ts +++ b/src/test/unittests/unittest/unittest.test.ts @@ -4,7 +4,10 @@ import * as path from 'path'; import { ConfigurationTarget } from 'vscode'; import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; import { CommandSource } from '../../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { + ITestManagerFactory, TestFile, + TestFunction, Tests, TestsToRun +} from '../../../client/unittests/common/types'; import { rootWorkspaceUri, updateSetting } from '../../common'; import { UnitTestIocContainer } from '../serviceRegistry'; import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; @@ -12,6 +15,7 @@ import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initiali const testFilesPath = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles'); const UNITTEST_TEST_FILES_PATH = path.join(testFilesPath, 'standard'); const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(testFilesPath, 'single'); +const UNITTEST_MULTI_TEST_FILE_PATH = path.join(testFilesPath, 'multi'); const defaultUnitTestArgs = [ '-v', '-s', @@ -60,4 +64,42 @@ suite('Unit Tests - unittest - discovery against actual python process', () => { assert.equal(tests.testSuites.length, 1, 'Incorrect number of test suites'); assert.equal(tests.testFiles.some(t => t.name === 'test_one.py' && t.nameToRun === 'test_one.Test_test1.test_A'), true, 'Test File not found'); }); + + test('Discover Tests (many test files, subdir included)', async () => { + await updateSetting('unitTest.unittestArgs', ['-s=./tests', '-p=test_*.py'], rootWorkspaceUri, configTarget); + const factory = ioc.serviceContainer.get(ITestManagerFactory); + const testManager = factory('unittest', rootWorkspaceUri, UNITTEST_MULTI_TEST_FILE_PATH); + const tests = await testManager.discoverTests(CommandSource.ui, true, true); + assert.equal(tests.testFiles.length, 3, 'Incorrect number of test files'); + assert.equal(tests.testFunctions.length, 9, 'Incorrect number of test functions'); + assert.equal(tests.testSuites.length, 3, 'Incorrect number of test suites'); + assert.equal(tests.testFiles.some(t => t.name === 'test_one.py' && t.nameToRun === 'test_one.Test_test1.test_A'), true, 'Test File one not found'); + assert.equal(tests.testFiles.some(t => t.name === 'test_two.py' && t.nameToRun === 'test_two.Test_test2.test_2A'), true, 'Test File two not found'); + assert.equal(tests.testFiles.some(t => t.name === 'test_three.py' && t.nameToRun === 'more_tests.test_three.Test_test3.test_3A'), true, 'Test File three not found'); + }); + + test('Run single test', async () => { + await updateSetting('unitTest.unittestArgs', ['-s=./tests', '-p=test_*.py'], rootWorkspaceUri, configTarget); + const factory = ioc.serviceContainer.get(ITestManagerFactory); + const testManager = factory('unittest', rootWorkspaceUri, UNITTEST_MULTI_TEST_FILE_PATH); + const testsDiscovered: Tests = await testManager.discoverTests(CommandSource.ui, true, true); + const testFile: TestFile | undefined = testsDiscovered.testFiles.find( + (value: TestFile) => value.nameToRun.endsWith('_3A') + ); + assert.notEqual(testFile, undefined, 'No test file suffixed with _3A in test files.'); + assert.equal(testFile!.suites.length, 1, 'Expected only 1 test suite in test file three.'); + const testFunc: TestFunction | undefined = testFile!.suites[0].functions.find( + (value: TestFunction) => value.name === 'test_3A' + ); + assert.notEqual(testFunc, undefined, 'No test in file test_three.py named test_3A'); + const testsToRun: TestsToRun = { + testFunction: [testFunc!] + }; + const testRunResult: Tests = await testManager.runTest(CommandSource.ui, testsToRun); + assert.equal(testRunResult.summary.failures + testRunResult.summary.passed + testRunResult.summary.skipped, 1, 'Expected to see only 1 test run in the summary for tests run.'); + assert.equal(testRunResult.summary.errors, 0, 'Unexpected: Test file ran with errors.'); + assert.equal(testRunResult.summary.failures, 0, 'Unexpected: Test has failed during test run.'); + assert.equal(testRunResult.summary.passed, 1, `Only one test should have passed during our test run. Instead, ${testRunResult.summary.passed} passed.`); + assert.equal(testRunResult.summary.skipped, 0, `Expected to have skipped 0 tests during this test-run. Instead, ${testRunResult.summary.skipped} where skipped.`); + }); });