Skip to content

Commit c7e2a83

Browse files
d3r3kkd3r3kk
authored
Disable consistently failing tests for specific OS and Python versions to get CI running clean. (microsoft#2944)
1 parent 68f8423 commit c7e2a83

10 files changed

Lines changed: 345 additions & 64 deletions

File tree

‎news/3 Code Health/2795.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Skip known failing tests for specific OS and Python version combinations to get CI running cleanly.

‎src/test/autocomplete/pep526.test.ts‎

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
1+
'use strict';
2+
13
import * as assert from 'assert';
24
import * as path from 'path';
35
import * as vscode from 'vscode';
4-
import { rootWorkspaceUri } from '../common';
5-
import { closeActiveWindows, initialize, initializeTest, IsLanguageServerTest } from '../initialize';
6+
import { OSType } from '../../client/common/utils/platform';
7+
import { isOs, isPythonVersion, rootWorkspaceUri } from '../common';
8+
import {
9+
closeActiveWindows, initialize,
10+
initializeTest, IsLanguageServerTest
11+
} from '../initialize';
612
import { UnitTestIocContainer } from '../unittests/serviceRegistry';
713

814
const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'autocomp');
@@ -40,7 +46,16 @@ suite('Autocomplete PEP 526', () => {
4046
ioc.registerProcessTypes();
4147
}
4248

43-
test('variable (abc:str)', async () => {
49+
test('variable (abc:str)', async function () {
50+
// This test has not been working for many months in Python 3.4 and 3.5 under
51+
// Windows and macOS.Tracked by #2545.
52+
if (isOs(OSType.Windows, OSType.OSX)) {
53+
if (await isPythonVersion('3.4', '3.5')) {
54+
// tslint:disable-next-line:no-invalid-this
55+
return this.skip();
56+
}
57+
}
58+
4459
const textDocument = await vscode.workspace.openTextDocument(filePep526);
4560
await vscode.window.showTextDocument(textDocument);
4661
assert(vscode.window.activeTextEditor, 'No active editor');
@@ -51,7 +66,16 @@ suite('Autocomplete PEP 526', () => {
5166
assert.notEqual(list!.items.filter(item => item.label === 'lower').length, 0, 'lower not found');
5267
});
5368

54-
test('variable (abc: str = "")', async () => {
69+
test('variable (abc: str = "")', async function () {
70+
// This test has not been working for many months in Python 3.4 and 3.5 under
71+
// Windows and macOS.Tracked by #2545.
72+
if (isOs(OSType.Windows, OSType.OSX)) {
73+
if (await isPythonVersion('3.4', '3.5')) {
74+
// tslint:disable-next-line:no-invalid-this
75+
return this.skip();
76+
}
77+
}
78+
5579
const textDocument = await vscode.workspace.openTextDocument(filePep526);
5680
await vscode.window.showTextDocument(textDocument);
5781
assert(vscode.window.activeTextEditor, 'No active editor');
@@ -62,7 +86,16 @@ suite('Autocomplete PEP 526', () => {
6286
assert.notEqual(list!.items.filter(item => item.label === 'lower').length, 0, 'lower not found');
6387
});
6488

65-
test('variable (abc = UNKNOWN # type: str)', async () => {
89+
test('variable (abc = UNKNOWN # type: str)', async function () {
90+
// This test has not been working for many months in Python 3.4 and 3.5 under
91+
// Windows and macOS.Tracked by #2545.
92+
if (isOs(OSType.Windows, OSType.OSX)) {
93+
if (await isPythonVersion('3.4', '3.5')) {
94+
// tslint:disable-next-line:no-invalid-this
95+
return this.skip();
96+
}
97+
}
98+
6699
const textDocument = await vscode.workspace.openTextDocument(filePep526);
67100
await vscode.window.showTextDocument(textDocument);
68101
assert(vscode.window.activeTextEditor, 'No active editor');
@@ -73,7 +106,16 @@ suite('Autocomplete PEP 526', () => {
73106
assert.notEqual(list!.items.filter(item => item.label === 'lower').length, 0, 'lower not found');
74107
});
75108

76-
test('class methods', async () => {
109+
test('class methods', async function () {
110+
// This test has not been working for many months in Python 3.4 and 3.5 under
111+
// Windows and macOS.Tracked by #2545.
112+
if (isOs(OSType.Windows, OSType.OSX)) {
113+
if (await isPythonVersion('3.4', '3.5')) {
114+
// tslint:disable-next-line:no-invalid-this
115+
return this.skip();
116+
}
117+
}
118+
77119
const textDocument = await vscode.workspace.openTextDocument(filePep526);
78120
await vscode.window.showTextDocument(textDocument);
79121
assert(vscode.window.activeTextEditor, 'No active editor');

‎src/test/common.ts‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,18 @@
1+
'use strict';
2+
13
import * as fs from 'fs-extra';
24
import * as path from 'path';
5+
import { coerce, SemVer } from 'semver';
36
import { ConfigurationTarget, Uri, workspace } from 'vscode';
47
import { PythonSettings } from '../client/common/configSettings';
58
import { EXTENSION_ROOT_DIR } from '../client/common/constants';
9+
import { traceError } from '../client/common/logger';
10+
import { BufferDecoder } from '../client/common/process/decoder';
11+
import { ProcessService } from '../client/common/process/proc';
12+
import { IProcessService } from '../client/common/process/types';
13+
import { getOSType, OSType } from '../client/common/utils/platform';
614
import { IS_MULTI_ROOT_TEST } from './initialize';
15+
716
export { sleep } from './core';
817

918
// tslint:disable:no-invalid-this no-any
@@ -128,3 +137,153 @@ function getPythonPath(): string {
128137
}
129138
return 'python';
130139
}
140+
141+
/**
142+
* Determine if the current platform is included in a list of platforms.
143+
*
144+
* @param {OSes} OSType[] List of operating system Ids to check within.
145+
* @return true if the current OS matches one from the list, false otherwise.
146+
*/
147+
export function isOs(...OSes: OSType[]): boolean {
148+
// get current OS
149+
const currentOS: OSType = getOSType();
150+
// compare and return
151+
if (OSes.indexOf(currentOS) === -1) {
152+
return false;
153+
}
154+
return true;
155+
}
156+
157+
/**
158+
* Get the current Python interpreter version.
159+
*
160+
* @param {procService} IProcessService Optionally specify the IProcessService implementation to use to execute with.
161+
* @return `SemVer` version of the Python interpreter, or `undefined` if an error occurs.
162+
*/
163+
export async function getPythonSemVer(procService?: IProcessService): Promise<SemVer | undefined> {
164+
const pythonProcRunner = procService ? procService : new ProcessService(new BufferDecoder());
165+
const pyVerArgs = ['-c', 'import sys;print("{0}.{1}.{2}".format(*sys.version_info[:3]))'];
166+
167+
return pythonProcRunner.exec(PYTHON_PATH, pyVerArgs)
168+
.then(strVersion => new SemVer(strVersion.stdout.trim()))
169+
.catch((err) => {
170+
// if the call fails this should make it loudly apparent.
171+
traceError('getPythonSemVer', err);
172+
return undefined;
173+
});
174+
}
175+
176+
/**
177+
* Match a given semver version specification with a list of loosely defined
178+
* version strings.
179+
*
180+
* Specify versions by their major version at minimum - the minor and patch
181+
* version numbers are optional.
182+
*
183+
* '3', '3.6', '3.6.6', are all vald and only the portions specified will be matched
184+
* against the current running Python interpreter version.
185+
*
186+
* Example scenarios:
187+
* '3' will match version 3.5.6, 3.6.4, 3.6.6, and 3.7.0.
188+
* '3.6' will match version 3.6.4 and 3.6.6.
189+
* '3.6.4' will match version 3.6.4 only.
190+
*
191+
* @param {version} SemVer the version to look for.
192+
* @param {searchVersions} string[] List of loosely-specified versions to match against.
193+
*/
194+
export function isVersionInList(version: SemVer, ...searchVersions: string[]): boolean {
195+
// see if the major/minor version matches any member of the skip-list.
196+
const isPresent = searchVersions.findIndex(ver => {
197+
const semverChecker = coerce(ver);
198+
if (semverChecker) {
199+
if (semverChecker.compare(version) === 0) {
200+
return true;
201+
} else {
202+
// compare all the parts of the version that we have, we know we have
203+
// at minimum the major version or semverChecker would be 'null'...
204+
const versionParts = ver.split('.');
205+
let matches = parseInt(versionParts[0], 10) === version.major;
206+
207+
if (matches && versionParts.length >= 2) {
208+
matches = parseInt(versionParts[1], 10) === version.minor;
209+
}
210+
211+
if (matches && versionParts.length >= 3) {
212+
matches = parseInt(versionParts[2], 10) === version.patch;
213+
}
214+
215+
return matches;
216+
}
217+
}
218+
return false;
219+
});
220+
221+
if (isPresent >= 0) {
222+
return true;
223+
}
224+
return false;
225+
}
226+
227+
/**
228+
* Determine if the Python interpreter version running in a given `IProcessService`
229+
* is in a selection of versions.
230+
*
231+
* You can specify versions by specifying the major version at minimum - the minor and
232+
* patch version numbers are optional.
233+
*
234+
* '3', '3.6', '3.6.6', are all vald and only the portions specified will be matched
235+
* against the current running Python interpreter version.
236+
*
237+
* Example scenarios:
238+
* '3' will match version 3.5.6, 3.6.4, 3.6.6, and 3.7.0.
239+
* '3.6' will match version 3.6.4 and 3.6.6.
240+
* '3.6.4' will match version 3.6.4 only.
241+
*
242+
* If you don't need to specify the environment (ie. the workspace) that the Python
243+
* interpreter is running under, use the simpler `isPythonVersion` instead.
244+
*
245+
* @param {skipForVersions} string[] List of versions of python that are to be skipped.
246+
* @param {resource} vscode.Uri Current workspace resource Uri or undefined.
247+
* @return true if the current Python version matches a version in the skip list, false otherwise.
248+
*/
249+
export async function isPythonVersionInProcess(procService?: IProcessService, ...versions: string[]): Promise<boolean> {
250+
// get the current python version major/minor
251+
const currentPyVersion = await getPythonSemVer(procService);
252+
if (currentPyVersion) {
253+
return isVersionInList(currentPyVersion, ...versions);
254+
} else {
255+
traceError(`Failed to determine the current Python version when comparing against list [${versions.join(', ')}].`);
256+
return false;
257+
}
258+
}
259+
260+
/**
261+
* Determine if the current interpreter version is in a given selection of versions.
262+
*
263+
* You can specify versions by using up to the first three semver parts of a python
264+
* version.
265+
*
266+
* '3', '3.6', '3.6.6', are all vald and only the portions specified will be matched
267+
* against the current running Python interpreter version.
268+
*
269+
* Example scenarios:
270+
* '3' will match version 3.5.6, 3.6.4, 3.6.6, and 3.7.0.
271+
* '3.6' will match version 3.6.4 and 3.6.6.
272+
* '3.6.4' will match version 3.6.4 only.
273+
*
274+
* If you need to specify the environment (ie. the workspace) that the Python
275+
* interpreter is running under, use `isPythonVersionInProcess` instead.
276+
*
277+
* @param {skipForVersions} string[] List of versions of python that are to be skipped.
278+
* @param {resource} vscode.Uri Current workspace resource Uri or undefined.
279+
* @return true if the current Python version matches a version in the skip list, false otherwise.
280+
*/
281+
export async function isPythonVersion(...versions: string[]): Promise<boolean> {
282+
const currentPyVersion = await getPythonSemVer();
283+
if (currentPyVersion) {
284+
return isVersionInList(currentPyVersion, ...versions);
285+
} else {
286+
traceError(`Failed to determine the current Python version when comparing against list [${versions.join(', ')}].`);
287+
return false;
288+
}
289+
}

‎src/test/common/process/proc.exec.test.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
'use strict';
5+
46
import { expect, use } from 'chai';
57
import * as chaiAsPromised from 'chai-as-promised';
68
import { CancellationTokenSource } from 'vscode';
79
import { PythonSettings } from '../../../client/common/configSettings';
810
import { BufferDecoder } from '../../../client/common/process/decoder';
911
import { ProcessService } from '../../../client/common/process/proc';
1012
import { StdErrError } from '../../../client/common/process/types';
13+
import { OSType } from '../../../client/common/utils/platform';
14+
import { isOs, isPythonVersion } from '../../common';
1115
import { initialize } from './../../initialize';
1216

1317
use(chaiAsPromised);
1418

1519
// tslint:disable-next-line:max-func-body-length
16-
suite('ProcessService', () => {
20+
suite('ProcessService Observable', () => {
1721
let pythonPath: string;
1822
suiteSetup(() => {
1923
pythonPath = PythonSettings.getInstance().pythonPath;
@@ -32,7 +36,14 @@ suite('ProcessService', () => {
3236
expect(result.stderr).to.equal(undefined, 'stderr not undefined');
3337
});
3438

35-
test('exec should output print unicode characters', async () => {
39+
test('exec should output print unicode characters', async function () {
40+
// This test has not been working for many months in Python 2.7 under
41+
// Windows. Tracked by #2546. (unicode under Py2.7 is tough!)
42+
if (isOs(OSType.Windows) && await isPythonVersion('2.7')) {
43+
// tslint:disable-next-line:no-invalid-this
44+
return this.skip();
45+
}
46+
3647
const procService = new ProcessService(new BufferDecoder());
3748
const printOutput = 'öä';
3849
const result = await procService.exec(pythonPath, ['-c', `print("${printOutput}")`]);

‎src/test/common/process/pythonProc.simple.multiroot.test.ts‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
'use strict';
5+
46
import { expect, use } from 'chai';
57
import * as chaiAsPromised from 'chai-as-promised';
68
import { execFile } from 'child_process';
@@ -18,14 +20,26 @@ import { IFileSystem, IPlatformService } from '../../../client/common/platform/t
1820
import { CurrentProcess } from '../../../client/common/process/currentProcess';
1921
import { registerTypes as processRegisterTypes } from '../../../client/common/process/serviceRegistry';
2022
import { IPythonExecutionFactory, StdErrError } from '../../../client/common/process/types';
21-
import { IConfigurationService, ICurrentProcess, IDisposableRegistry, IPathUtils, IsWindows } from '../../../client/common/types';
23+
import {
24+
IConfigurationService, ICurrentProcess,
25+
IDisposableRegistry, IPathUtils, IsWindows
26+
} from '../../../client/common/types';
2227
import { IS_WINDOWS } from '../../../client/common/util';
23-
import { registerTypes as variablesRegisterTypes } from '../../../client/common/variables/serviceRegistry';
28+
import { OSType } from '../../../client/common/utils/platform';
29+
import {
30+
registerTypes as variablesRegisterTypes
31+
} from '../../../client/common/variables/serviceRegistry';
2432
import { ServiceContainer } from '../../../client/ioc/container';
2533
import { ServiceManager } from '../../../client/ioc/serviceManager';
2634
import { IServiceContainer } from '../../../client/ioc/types';
27-
import { clearPythonPathInWorkspaceFolder } from '../../common';
28-
import { closeActiveWindows, initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize';
35+
import {
36+
clearPythonPathInWorkspaceFolder, isOs,
37+
isPythonVersion
38+
} from '../../common';
39+
import {
40+
closeActiveWindows, initialize, initializeTest,
41+
IS_MULTI_ROOT_TEST
42+
} from './../../initialize';
2943

3044
use(chaiAsPromised);
3145

@@ -90,7 +104,14 @@ suite('PythonExecutableService', () => {
90104
await expect(promise).to.eventually.be.rejectedWith(StdErrError);
91105
});
92106

93-
test('Importing with a valid PYTHONPATH from .env file should succeed', async () => {
107+
test('Importing with a valid PYTHONPATH from .env file should succeed', async function () {
108+
// This test has not been working for many months in Python 2.7 under
109+
// Windows. Tracked by #2547.
110+
if (isOs(OSType.Windows) && await isPythonVersion('2.7')) {
111+
// tslint:disable-next-line:no-invalid-this
112+
return this.skip();
113+
}
114+
94115
await configService.updateSetting('envFile', undefined, workspace4PyFile, ConfigurationTarget.WorkspaceFolder);
95116
const pythonExecService = await pythonExecFactory.create({ resource: workspace4PyFile });
96117
const promise = pythonExecService.exec([workspace4PyFile.fsPath], { cwd: path.dirname(workspace4PyFile.fsPath), throwOnStdErr: true });

‎src/test/format/extension.format.test.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { IProcessServiceFactory, IPythonExecutionFactory } from '../../client/co
55
import { AutoPep8Formatter } from '../../client/formatters/autoPep8Formatter';
66
import { BlackFormatter } from '../../client/formatters/blackFormatter';
77
import { YapfFormatter } from '../../client/formatters/yapfFormatter';
8-
import { PythonVersionInformation } from '../../client/unittests/common/types';
8+
import { isPythonVersionInProcess } from '../common';
99
import { closeActiveWindows, initialize, initializeTest } from '../initialize';
1010
import { MockProcessService } from '../mocks/proc';
1111
import { compareFiles } from '../textUtils';
@@ -121,9 +121,10 @@ suite('Formatting', () => {
121121
});
122122
// tslint:disable-next-line:no-function-expression
123123
test('Black', async function () {
124-
const pyVersion: PythonVersionInformation = await ioc.getPythonMajorMinorVersion(Uri.parse(blackFileToFormat));
125-
126-
if (pyVersion && (pyVersion.major < 3 || (pyVersion.major === 3 && pyVersion.minor < 6))) {
124+
const processService = await ioc.serviceContainer.get<IProcessServiceFactory>(IProcessServiceFactory)
125+
.create(Uri.file(workspaceRootPath));
126+
if (await isPythonVersionInProcess(processService, '2.7', '3.4', '3.5')) {
127+
// Skip for versions of python below 3.6, as Black doesn't support them at all.
127128
// tslint:disable-next-line:no-invalid-this
128129
return this.skip();
129130
}

0 commit comments

Comments
 (0)