Skip to content

Commit 0b14333

Browse files
author
Eric Snow
authored
Separate Python env from exec. (#10883)
For #10681 The key change here is the separation (under src/client/common/process) of Python environment helpers from running Python processes. This helps simplify later changes needed for #10681, as well as other code health benefits. A small related change: adding PythonExecutionInfo.python (which helps simplify composition of exec args in some situations). There should be zero change in behavior.
1 parent c48b306 commit 0b14333

32 files changed

Lines changed: 741 additions & 613 deletions

‎src/client/common/process/condaExecutionService.ts‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎src/client/common/process/pythonDaemon.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ export class PythonDaemonExecutionService implements IPythonDaemonExecutionServi
114114
return this.pythonExecutionService.getExecutablePath();
115115
}
116116
}
117-
public getExecutionInfo(args: string[]): PythonExecutionInfo {
118-
return this.pythonExecutionService.getExecutionInfo(args);
117+
public getExecutionInfo(pythonArgs?: string[]): PythonExecutionInfo {
118+
return this.pythonExecutionService.getExecutionInfo(pythonArgs);
119119
}
120120
public async isModuleInstalled(moduleName: string): Promise<boolean> {
121121
try {

‎src/client/common/process/pythonDaemonPool.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ export class PythonDaemonExecutionServicePool implements IPythonDaemonExecutionS
9797
const msg = { args: ['getExecutablePath'] };
9898
return this.wrapCall((daemon) => daemon.getExecutablePath(), msg);
9999
}
100-
public getExecutionInfo(args: string[]): PythonExecutionInfo {
101-
return this.pythonExecutionService.getExecutionInfo(args);
100+
public getExecutionInfo(pythonArgs?: string[]): PythonExecutionInfo {
101+
return this.pythonExecutionService.getExecutionInfo(pythonArgs);
102102
}
103103
public async isModuleInstalled(moduleName: string): Promise<boolean> {
104104
const msg = { args: ['-m', moduleName] };

‎src/client/common/process/pythonEnvironment.ts‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import * as path from 'path';
5+
import { CondaEnvironmentInfo } from '../../interpreter/contracts';
6+
import { EXTENSION_ROOT_DIR } from '../constants';
7+
import { traceError, traceInfo } from '../logger';
8+
import { IFileSystem } from '../platform/types';
9+
import { Architecture } from '../utils/platform';
10+
import { parsePythonVersion } from '../utils/version';
11+
import {
12+
ExecutionResult,
13+
InterpreterInfomation,
14+
IProcessService,
15+
PythonExecutionInfo,
16+
PythonVersionInfo,
17+
ShellOptions,
18+
SpawnOptions
19+
} from './types';
20+
21+
function getExecutionInfo(python: string[], pythonArgs: string[]): PythonExecutionInfo {
22+
const args = python.slice(1);
23+
args.push(...pythonArgs);
24+
return { command: python[0], args, python };
25+
}
26+
27+
class PythonEnvironment {
28+
private cachedInterpreterInformation: InterpreterInfomation | undefined | null = null;
29+
30+
constructor(
31+
protected readonly pythonPath: string,
32+
// "deps" is the externally defined functionality used by the class.
33+
protected readonly deps: {
34+
getPythonArgv(python: string): string[];
35+
getObservablePythonArgv(python: string): string[];
36+
isValidExecutable(python: string): Promise<boolean>;
37+
// from ProcessService:
38+
exec(file: string, args: string[]): Promise<ExecutionResult<string>>;
39+
shellExec(command: string, timeout: number): Promise<ExecutionResult<string>>;
40+
}
41+
) {}
42+
43+
public getExecutionInfo(pythonArgs: string[] = []): PythonExecutionInfo {
44+
const python = this.deps.getPythonArgv(this.pythonPath);
45+
return getExecutionInfo(python, pythonArgs);
46+
}
47+
public getExecutionObservableInfo(pythonArgs: string[] = []): PythonExecutionInfo {
48+
const python = this.deps.getObservablePythonArgv(this.pythonPath);
49+
return getExecutionInfo(python, pythonArgs);
50+
}
51+
52+
public async getInterpreterInformation(): Promise<InterpreterInfomation | undefined> {
53+
if (this.cachedInterpreterInformation === null) {
54+
this.cachedInterpreterInformation = await this.getInterpreterInformationImpl();
55+
}
56+
return this.cachedInterpreterInformation;
57+
}
58+
59+
public async getExecutablePath(): Promise<string> {
60+
// If we've passed the python file, then return the file.
61+
// This is because on mac if using the interpreter /usr/bin/python2.7 we can get a different value for the path
62+
if (await this.deps.isValidExecutable(this.pythonPath)) {
63+
return this.pythonPath;
64+
}
65+
66+
const { command, args } = this.getExecutionInfo(['-c', 'import sys;print(sys.executable)']);
67+
const proc = await this.deps.exec(command, args);
68+
return proc.stdout.trim();
69+
}
70+
71+
public async isModuleInstalled(moduleName: string): Promise<boolean> {
72+
const { command, args } = this.getExecutionInfo(['-c', `import ${moduleName}`]);
73+
try {
74+
await this.deps.exec(command, args);
75+
} catch {
76+
return false;
77+
}
78+
return true;
79+
}
80+
81+
private async getInterpreterInformationImpl(): Promise<InterpreterInfomation | undefined> {
82+
try {
83+
const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'interpreterInfo.py');
84+
const { command, args } = this.getExecutionInfo([file]);
85+
const argv = [command, ...args];
86+
87+
// Concat these together to make a set of quoted strings
88+
const quoted = argv.reduce((p, c) => (p ? `${p} "${c}"` : `"${c.replace('\\', '\\\\')}"`), '');
89+
90+
// Try shell execing the command, followed by the arguments. This will make node kill the process if it
91+
// takes too long.
92+
// Sometimes the python path isn't valid, timeout if that's the case.
93+
// See these two bugs:
94+
// https://github.com/microsoft/vscode-python/issues/7569
95+
// https://github.com/microsoft/vscode-python/issues/7760
96+
const result = await this.deps.shellExec(quoted, 15000);
97+
if (result.stderr) {
98+
traceError(`Failed to parse interpreter information for ${argv} stderr: ${result.stderr}`);
99+
return;
100+
}
101+
let json: {
102+
versionInfo: PythonVersionInfo;
103+
sysPrefix: string;
104+
sysVersion: string;
105+
is64Bit: boolean;
106+
};
107+
try {
108+
json = JSON.parse(result.stdout);
109+
} catch (ex) {
110+
throw Error(`${argv} returned bad JSON (${result.stdout}) (${ex})`);
111+
}
112+
traceInfo(`Found interpreter for ${argv}`);
113+
const versionValue =
114+
json.versionInfo.length === 4
115+
? `${json.versionInfo.slice(0, 3).join('.')}-${json.versionInfo[3]}`
116+
: json.versionInfo.join('.');
117+
return {
118+
architecture: json.is64Bit ? Architecture.x64 : Architecture.x86,
119+
path: this.pythonPath,
120+
version: parsePythonVersion(versionValue),
121+
sysVersion: json.sysVersion,
122+
sysPrefix: json.sysPrefix
123+
};
124+
} catch (ex) {
125+
traceError(`Failed to get interpreter information for '${this.pythonPath}'`, ex);
126+
}
127+
}
128+
}
129+
130+
function createDeps(
131+
isValidExecutable: (filename: string) => Promise<boolean>,
132+
pythonArgv: string[] | undefined,
133+
observablePythonArgv: string[] | undefined,
134+
// from ProcessService:
135+
exec: (file: string, args: string[], options?: SpawnOptions) => Promise<ExecutionResult<string>>,
136+
shellExec: (command: string, options?: ShellOptions) => Promise<ExecutionResult<string>>
137+
) {
138+
return {
139+
getPythonArgv: (python: string) => pythonArgv || [python],
140+
getObservablePythonArgv: (python: string) => observablePythonArgv || [python],
141+
isValidExecutable,
142+
exec: async (cmd: string, args: string[]) => exec(cmd, args, { throwOnStdErr: true }),
143+
shellExec: async (text: string, timeout: number) => shellExec(text, { timeout })
144+
};
145+
}
146+
147+
export function createPythonEnv(
148+
pythonPath: string,
149+
// These are used to generate the deps.
150+
procs: IProcessService,
151+
fs: IFileSystem
152+
): PythonEnvironment {
153+
const deps = createDeps(
154+
async (filename) => fs.fileExists(filename),
155+
// We use the default: [pythonPath].
156+
undefined,
157+
undefined,
158+
(file, args, opts) => procs.exec(file, args, opts),
159+
(command, opts) => procs.shellExec(command, opts)
160+
);
161+
return new PythonEnvironment(pythonPath, deps);
162+
}
163+
164+
export function createCondaEnv(
165+
condaFile: string,
166+
condaInfo: CondaEnvironmentInfo,
167+
pythonPath: string,
168+
// These are used to generate the deps.
169+
procs: IProcessService,
170+
fs: IFileSystem
171+
): PythonEnvironment {
172+
const runArgs = ['run'];
173+
if (condaInfo.name === '') {
174+
runArgs.push('-p', condaInfo.path);
175+
} else {
176+
runArgs.push('-n', condaInfo.name);
177+
}
178+
const pythonArgv = [condaFile, ...runArgs, 'python'];
179+
const deps = createDeps(
180+
async (filename) => fs.fileExists(filename),
181+
pythonArgv,
182+
// tslint:disable-next-line:no-suspicious-comment
183+
// TODO(gh-8473): Use pythonArgv here once 'conda run' can be
184+
// run without buffering output.
185+
undefined,
186+
(file, args, opts) => procs.exec(file, args, opts),
187+
(command, opts) => procs.shellExec(command, opts)
188+
);
189+
return new PythonEnvironment(pythonPath, deps);
190+
}
191+
192+
export function createWindowsStoreEnv(
193+
pythonPath: string,
194+
// These are used to generate the deps.
195+
procs: IProcessService
196+
): PythonEnvironment {
197+
const deps = createDeps(
198+
/**
199+
* With windows store python apps, we have generally use the
200+
* symlinked python executable. The actual file is not accessible
201+
* by the user due to permission issues (& rest of exension fails
202+
* when using that executable). Hence lets not resolve the
203+
* executable using sys.executable for windows store python
204+
* interpreters.
205+
*/
206+
async (_f: string) => true,
207+
// We use the default: [pythonPath].
208+
undefined,
209+
undefined,
210+
(file, args, opts) => procs.exec(file, args, opts),
211+
(command, opts) => procs.shellExec(command, opts)
212+
);
213+
return new PythonEnvironment(pythonPath, deps);
214+
}

‎src/client/common/process/pythonExecutionFactory.ts‎

Lines changed: 46 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@ import { gte } from 'semver';
55

66
import { Uri } from 'vscode';
77
import { IEnvironmentActivationService } from '../../interpreter/activation/types';
8-
import { ICondaService, IInterpreterService } from '../../interpreter/contracts';
8+
import { CondaEnvironmentInfo, ICondaService, IInterpreterService } from '../../interpreter/contracts';
99
import { WindowsStoreInterpreter } from '../../interpreter/locators/services/windowsStoreInterpreter';
1010
import { IWindowsStoreInterpreter } from '../../interpreter/locators/types';
1111
import { IServiceContainer } from '../../ioc/types';
1212
import { sendTelemetryEvent } from '../../telemetry';
1313
import { EventName } from '../../telemetry/constants';
1414
import { traceError } from '../logger';
15+
import { IFileSystem } from '../platform/types';
1516
import { IConfigurationService, IDisposableRegistry } from '../types';
16-
import { CondaExecutionService } from './condaExecutionService';
1717
import { ProcessService } from './proc';
1818
import { PythonDaemonExecutionServicePool } from './pythonDaemonPool';
19-
import { PythonExecutionService } from './pythonProcess';
19+
import { createCondaEnv, createPythonEnv, createWindowsStoreEnv } from './pythonEnvironment';
20+
import { createPythonProcessService } from './pythonProcess';
2021
import {
2122
DaemonExecutionFactoryCreationOptions,
2223
ExecutionFactoryCreateWithEnvironmentOptions,
@@ -29,7 +30,6 @@ import {
2930
IPythonExecutionFactory,
3031
IPythonExecutionService
3132
} from './types';
32-
import { WindowsStorePythonProcess } from './windowsStorePythonProcess';
3333

3434
// Minimum version number of conda required to be able to use 'conda run'
3535
export const CONDA_RUN_VERSION = '4.6.0';
@@ -54,16 +54,15 @@ export class PythonExecutionFactory implements IPythonExecutionFactory {
5454
const processLogger = this.serviceContainer.get<IProcessLogger>(IProcessLogger);
5555
processService.on('exec', processLogger.logProcess.bind(processLogger));
5656

57-
if (this.windowsStoreInterpreter.isWindowsStoreInterpreter(pythonPath)) {
58-
return new WindowsStorePythonProcess(
59-
this.serviceContainer,
60-
processService,
61-
pythonPath,
62-
this.windowsStoreInterpreter
63-
);
64-
}
65-
return new PythonExecutionService(this.serviceContainer, processService, pythonPath);
57+
return createPythonService(
58+
pythonPath,
59+
processService,
60+
this.serviceContainer.get<IFileSystem>(IFileSystem),
61+
undefined,
62+
this.windowsStoreInterpreter.isWindowsStoreInterpreter(pythonPath)
63+
);
6664
}
65+
6766
public async createDaemon(options: DaemonExecutionFactoryCreationOptions): Promise<IPythonExecutionService> {
6867
const pythonPath = options.pythonPath
6968
? options.pythonPath
@@ -143,15 +142,15 @@ export class PythonExecutionFactory implements IPythonExecutionFactory {
143142
processService.on('exec', processLogger.logProcess.bind(processLogger));
144143
this.serviceContainer.get<IDisposableRegistry>(IDisposableRegistry).push(processService);
145144

146-
return new PythonExecutionService(this.serviceContainer, processService, pythonPath);
145+
return createPythonService(pythonPath, processService, this.serviceContainer.get<IFileSystem>(IFileSystem));
147146
}
148147
// Not using this function for now because there are breaking issues with conda run (conda 4.8, PVSC 2020.1).
149148
// See https://github.com/microsoft/vscode-python/issues/9490
150149
public async createCondaExecutionService(
151150
pythonPath: string,
152151
processService?: IProcessService,
153152
resource?: Uri
154-
): Promise<CondaExecutionService | undefined> {
153+
): Promise<IPythonExecutionService | undefined> {
155154
const processServicePromise = processService
156155
? Promise.resolve(processService)
157156
: this.processServiceFactory.create(resource);
@@ -169,15 +168,42 @@ export class PythonExecutionFactory implements IPythonExecutionFactory {
169168
procService.on('exec', processLogger.logProcess.bind(processLogger));
170169
this.serviceContainer.get<IDisposableRegistry>(IDisposableRegistry).push(procService);
171170
}
172-
return new CondaExecutionService(
173-
this.serviceContainer,
174-
procService,
171+
return createPythonService(
175172
pythonPath,
176-
condaFile,
177-
condaEnvironment
173+
procService,
174+
this.serviceContainer.get<IFileSystem>(IFileSystem),
175+
// This is what causes a CondaEnvironment to be returned:
176+
[condaFile, condaEnvironment]
178177
);
179178
}
180179

181180
return Promise.resolve(undefined);
182181
}
183182
}
183+
184+
function createPythonService(
185+
pythonPath: string,
186+
procService: IProcessService,
187+
fs: IFileSystem,
188+
conda?: [string, CondaEnvironmentInfo],
189+
isWindowsStore?: boolean
190+
): IPythonExecutionService {
191+
let env = createPythonEnv(pythonPath, procService, fs);
192+
if (conda) {
193+
const [condaPath, condaInfo] = conda;
194+
env = createCondaEnv(condaPath, condaInfo, pythonPath, procService, fs);
195+
} else if (isWindowsStore) {
196+
env = createWindowsStoreEnv(pythonPath, procService);
197+
}
198+
const procs = createPythonProcessService(procService, env);
199+
return {
200+
getInterpreterInformation: () => env.getInterpreterInformation(),
201+
getExecutablePath: () => env.getExecutablePath(),
202+
isModuleInstalled: (m) => env.isModuleInstalled(m),
203+
getExecutionInfo: (a) => env.getExecutionInfo(a),
204+
execObservable: (a, o) => procs.execObservable(a, o),
205+
execModuleObservable: (m, a, o) => procs.execModuleObservable(m, a, o),
206+
exec: (a, o) => procs.exec(a, o),
207+
execModule: (m, a, o) => procs.execModule(m, a, o)
208+
};
209+
}

0 commit comments

Comments
 (0)