Skip to content

Commit 06c9b9e

Browse files
author
Kartik Raj
authored
Added Pyenv file system watcher (microsoft#14792)
* Added workspace virtual env watcher * Fix tests * Remove option * Mege the two tests together * Skip * Try this out * Let's try this fix * Clean up * Fix lint errors * Added Pyenv file system watcher * Consolidate similar watcher tests together * Fix unit tests * Consolidate all the tests together * Remove redundant eslint rule * Provide an option to check for environment kind * Code reviews * Simplify * Replace any with unknown * Revert unknown to any
1 parent 92567af commit 06c9b9e

15 files changed

Lines changed: 302 additions & 301 deletions

File tree

‎src/client/pythonEnvironments/discovery/locators/services/globalVirtualEnvronmentLocator.ts‎

Lines changed: 21 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,6 @@ async function getVirtualEnvKind(interpreterPath: string): Promise<PythonEnvKind
8080
* Finds and resolves virtual environments created in known global locations.
8181
*/
8282
class GlobalVirtualEnvironmentLocator extends FSWatchingLocator {
83-
private virtualEnvKinds = [
84-
PythonEnvKind.Venv,
85-
PythonEnvKind.VirtualEnv,
86-
PythonEnvKind.VirtualEnvWrapper,
87-
PythonEnvKind.Pipenv,
88-
];
89-
9083
constructor(private readonly searchDepth?: number) {
9184
super(getGlobalVirtualEnvDirs, getVirtualEnvKind, {
9285
// Note detecting kind of virtual env depends on the file structure around the
@@ -102,7 +95,7 @@ class GlobalVirtualEnvironmentLocator extends FSWatchingLocator {
10295
// interpreters
10396
const searchDepth = this.searchDepth ?? DEFAULT_SEARCH_DEPTH;
10497

105-
async function* iterator(virtualEnvKinds: PythonEnvKind[]) {
98+
async function* iterator() {
10699
const envRootDirs = await getGlobalVirtualEnvDirs();
107100
const envGenerators = envRootDirs.map((envRootDir) => {
108101
async function* generator() {
@@ -122,19 +115,15 @@ class GlobalVirtualEnvironmentLocator extends FSWatchingLocator {
122115
const kind = await getVirtualEnvKind(env);
123116

124117
const timeData = await getFileInfo(env);
125-
if (virtualEnvKinds.includes(kind)) {
126-
traceVerbose(`Global Virtual Environment: [added] ${env}`);
127-
const envInfo = buildEnvInfo({
128-
kind,
129-
executable: env,
130-
version: UNKNOWN_PYTHON_VERSION,
131-
});
132-
envInfo.executable.ctime = timeData.ctime;
133-
envInfo.executable.mtime = timeData.mtime;
134-
yield envInfo;
135-
} else {
136-
traceVerbose(`Global Virtual Environment: [skipped] ${env}`);
137-
}
118+
traceVerbose(`Global Virtual Environment: [added] ${env}`);
119+
const envInfo = buildEnvInfo({
120+
kind,
121+
executable: env,
122+
version: UNKNOWN_PYTHON_VERSION,
123+
});
124+
envInfo.executable.ctime = timeData.ctime;
125+
envInfo.executable.mtime = timeData.mtime;
126+
yield envInfo;
138127
} else {
139128
traceVerbose(`Global Virtual Environment: [skipped] ${env}`);
140129
}
@@ -146,27 +135,26 @@ class GlobalVirtualEnvironmentLocator extends FSWatchingLocator {
146135
yield* iterable(chain(envGenerators));
147136
}
148137

149-
return iterator(this.virtualEnvKinds);
138+
return iterator();
150139
}
151140

141+
// eslint-disable-next-line class-methods-use-this
152142
public async resolveEnv(env: string | PythonEnvInfo): Promise<PythonEnvInfo | undefined> {
153143
const executablePath = typeof env === 'string' ? env : env.executable.filename;
154144
if (await pathExists(executablePath)) {
155145
// We should extract the kind here to avoid doing is*Environment()
156146
// check multiple times. Those checks are file system heavy and
157147
// we can use the kind to determine this anyway.
158148
const kind = await getVirtualEnvKind(executablePath);
159-
if (this.virtualEnvKinds.includes(kind)) {
160-
const timeData = await getFileInfo(executablePath);
161-
const envInfo = buildEnvInfo({
162-
kind,
163-
version: UNKNOWN_PYTHON_VERSION,
164-
executable: executablePath,
165-
});
166-
envInfo.executable.ctime = timeData.ctime;
167-
envInfo.executable.mtime = timeData.mtime;
168-
return envInfo;
169-
}
149+
const timeData = await getFileInfo(executablePath);
150+
const envInfo = buildEnvInfo({
151+
kind,
152+
version: UNKNOWN_PYTHON_VERSION,
153+
executable: executablePath,
154+
});
155+
envInfo.executable.ctime = timeData.ctime;
156+
envInfo.executable.mtime = timeData.mtime;
157+
return envInfo;
170158
}
171159
return undefined;
172160
}

‎src/client/pythonEnvironments/discovery/locators/services/pyenvLocator.ts‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ import {
99
PythonEnvInfo, PythonEnvKind,
1010
} from '../../../base/info';
1111
import { buildEnvInfo } from '../../../base/info/env';
12-
import { IDisposableLocator, IPythonEnvsIterator, Locator } from '../../../base/locator';
12+
import { IDisposableLocator, IPythonEnvsIterator } from '../../../base/locator';
13+
import { FSWatchingLocator } from '../../../base/locators/lowLevel/fsWatchingLocator';
1314
import {
1415
getEnvironmentDirFromPath, getInterpreterPathFromDir, getPythonVersionFromPath,
1516
} from '../../../common/commonUtils';
@@ -32,6 +33,10 @@ function getPyenvDir(): string {
3233
return pyenvDir;
3334
}
3435

36+
function getPyenvVersionsDir(): string {
37+
return path.join(getPyenvDir(), 'versions');
38+
}
39+
3540
/**
3641
* Checks if the given interpreter belongs to a pyenv based environment.
3742
* @param {string} interpreterPath: Absolute path to the python interpreter.
@@ -238,7 +243,7 @@ export function parsePyenvVersion(str:string): Promise<IPyenvVersionStrings|unde
238243
* best effort at identifying the versions and distribution information.
239244
*/
240245
async function* getPyenvEnvironments(): AsyncIterableIterator<PythonEnvInfo> {
241-
const pyenvVersionDir = path.join(getPyenvDir(), 'versions');
246+
const pyenvVersionDir = getPyenvVersionsDir();
242247

243248
const subDirs = getSubDirs(pyenvVersionDir);
244249
for await (const subDir of subDirs) {
@@ -293,7 +298,14 @@ async function* getPyenvEnvironments(): AsyncIterableIterator<PythonEnvInfo> {
293298
}
294299
}
295300

296-
class PyenvLocator extends Locator {
301+
class PyenvLocator extends FSWatchingLocator {
302+
constructor() {
303+
super(
304+
getPyenvVersionsDir,
305+
async () => PythonEnvKind.Pyenv,
306+
);
307+
}
308+
297309
// eslint-disable-next-line class-methods-use-this
298310
public iterEnvs(): IPythonEnvsIterator {
299311
return getPyenvEnvironments();
@@ -329,7 +341,8 @@ class PyenvLocator extends Locator {
329341
}
330342
}
331343

332-
export function createPyenvLocator(): Promise<IDisposableLocator> {
344+
export async function createPyenvLocator(): Promise<IDisposableLocator> {
333345
const locator = new PyenvLocator();
346+
await locator.initialize();
334347
return Promise.resolve(locator);
335348
}

‎src/test/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator.testvirtualenvs.ts‎

Lines changed: 6 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -1,187 +1,16 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4-
// eslint-disable-next-line max-classes-per-file
5-
import { assert } from 'chai';
6-
import * as fs from 'fs-extra';
74
import * as path from 'path';
8-
import { traceWarning } from '../../../../../client/common/logger';
9-
import { FileChangeType } from '../../../../../client/common/platform/fileSystemWatcher';
10-
import { createDeferred, Deferred, sleep } from '../../../../../client/common/utils/async';
11-
import { getOSType, OSType } from '../../../../../client/common/utils/platform';
12-
import { IDisposableLocator } from '../../../../../client/pythonEnvironments/base/locator';
135
import { createWorkspaceVirtualEnvLocator } from '../../../../../client/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator';
14-
import { getEnvs } from '../../../../../client/pythonEnvironments/base/locatorUtils';
15-
import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher';
16-
import { getInterpreterPathFromDir } from '../../../../../client/pythonEnvironments/common/commonUtils';
17-
import { arePathsSame } from '../../../../../client/pythonEnvironments/common/externalDependencies';
18-
import { deleteFiles, PYTHON_PATH } from '../../../../common';
19-
import { TEST_TIMEOUT } from '../../../../constants';
206
import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants';
21-
import { run } from '../../../discovery/locators/envTestUtils';
22-
23-
class WorkspaceVenvs {
24-
constructor(private readonly root: string, private readonly prefix = '.virtual') { }
25-
26-
public async create(name: string): Promise<string> {
27-
const envName = this.resolve(name);
28-
const argv = [PYTHON_PATH.fileToCommandArgument(), '-m', 'virtualenv', envName];
29-
try {
30-
await run(argv, { cwd: this.root });
31-
} catch (err) {
32-
throw new Error(`Failed to create Env ${path.basename(envName)} Error: ${err}`);
33-
}
34-
const dirToLookInto = path.join(this.root, envName);
35-
const filename = await getInterpreterPathFromDir(dirToLookInto);
36-
if (!filename) {
37-
throw new Error(`No environment to update exists in ${dirToLookInto}`);
38-
}
39-
return filename;
40-
}
41-
42-
/**
43-
* Creates a dummy environment by creating a fake executable.
44-
* @param name environment suffix name to create
45-
*/
46-
public async createDummyEnv(name: string): Promise<string> {
47-
const envName = this.resolve(name);
48-
const filepath = path.join(this.root, envName, getOSType() === OSType.Windows ? 'python.exe' : 'python');
49-
try {
50-
await fs.createFile(filepath);
51-
} catch (err) {
52-
throw new Error(`Failed to create python executable ${filepath}, Error: ${err}`);
53-
}
54-
return filepath;
55-
}
56-
57-
// eslint-disable-next-line class-methods-use-this
58-
public async update(filename: string): Promise<void> {
59-
try {
60-
await fs.writeFile(filename, 'Environment has been updated');
61-
} catch (err) {
62-
throw new Error(`Failed to update Workspace virtualenv executable ${filename}, Error: ${err}`);
63-
}
64-
}
65-
66-
// eslint-disable-next-line class-methods-use-this
67-
public async delete(filename: string): Promise<void> {
68-
try {
69-
await fs.remove(filename);
70-
} catch (err) {
71-
traceWarning(`Failed to clean up ${filename}`);
72-
}
73-
}
74-
75-
public async cleanUp() {
76-
const globPattern = path.join(this.root, `${this.prefix}*`);
77-
await deleteFiles(globPattern);
78-
}
79-
80-
private resolve(name: string): string {
81-
// Ensure env is random to avoid conflicts in tests (corrupting test data)
82-
const now = new Date().getTime().toString().substr(-8);
83-
return `${this.prefix}${name}${now}`;
84-
}
85-
}
7+
import { locatorFactoryFuncType, testLocatorWatcher } from '../../../discovery/locators/watcherTestUtils';
868

879
suite('WorkspaceVirtualEnvironment Locator', async () => {
8810
const testWorkspaceFolder = path.join(TEST_LAYOUT_ROOT, 'workspace', 'folder1');
89-
const workspaceVenvs = new WorkspaceVenvs(testWorkspaceFolder);
90-
let locator: IDisposableLocator;
91-
92-
async function waitForChangeToBeDetected(deferred: Deferred<void>) {
93-
const timeout = setTimeout(
94-
() => {
95-
clearTimeout(timeout);
96-
deferred.reject(new Error('Environment not detected'));
97-
},
98-
TEST_TIMEOUT,
99-
);
100-
await deferred.promise;
101-
}
102-
103-
async function isLocated(executable: string): Promise<boolean> {
104-
const items = await getEnvs(locator.iterEnvs());
105-
return items.some((item) => arePathsSame(item.executable.filename, executable));
106-
}
107-
108-
suiteSetup(async () => workspaceVenvs.cleanUp());
109-
110-
async function setupLocator(onChanged: (e: PythonEnvsChangedEvent) => Promise<void>) {
111-
locator = await createWorkspaceVirtualEnvLocator(testWorkspaceFolder);
112-
// Wait for watchers to get ready
113-
await sleep(1000);
114-
locator.onChanged(onChanged);
115-
}
116-
117-
teardown(async () => {
118-
await workspaceVenvs.cleanUp();
119-
locator.dispose();
120-
});
121-
122-
test('Detect a new environment', async () => {
123-
let actualEvent: PythonEnvsChangedEvent;
124-
const deferred = createDeferred<void>();
125-
await setupLocator(async (e) => {
126-
actualEvent = e;
127-
deferred.resolve();
128-
});
129-
130-
const executable = await workspaceVenvs.create('one');
131-
await waitForChangeToBeDetected(deferred);
132-
const isFound = await isLocated(executable);
133-
134-
assert.ok(isFound);
135-
// Detecting kind of virtual env depends on the file structure around the executable, so we need to wait before
136-
// attempting to verify it. Omitting that check as we can never deterministically say when it's ready to check.
137-
assert.deepEqual(actualEvent!.type, FileChangeType.Created, 'Wrong event emitted');
138-
});
139-
140-
test('Detect when an environment has been deleted', async () => {
141-
let actualEvent: PythonEnvsChangedEvent;
142-
const deferred = createDeferred<void>();
143-
const executable = await workspaceVenvs.create('one');
144-
// Wait before the change event has been sent. If both operations occur almost simultaneously no event is sent.
145-
await sleep(100);
146-
await setupLocator(async (e) => {
147-
actualEvent = e;
148-
deferred.resolve();
149-
});
150-
151-
// VSCode API has a limitation where it fails to fire event when environment folder is deleted directly:
152-
// https://github.com/microsoft/vscode/issues/110923
153-
// Using chokidar directly in tests work, but it has permission issues on Windows that you cannot delete a
154-
// folder if it has a subfolder that is being watched inside: https://github.com/paulmillr/chokidar/issues/422
155-
// Hence we test directly deleting the executable, and not the whole folder using `workspaceVenvs.cleanUp()`.
156-
await workspaceVenvs.delete(executable);
157-
await waitForChangeToBeDetected(deferred);
158-
const isFound = await isLocated(executable);
159-
160-
assert.notOk(isFound);
161-
assert.deepEqual(actualEvent!.type, FileChangeType.Deleted, 'Wrong event emitted');
162-
});
163-
164-
test('Detect when an environment has been updated', async () => {
165-
let actualEvent: PythonEnvsChangedEvent;
166-
const deferred = createDeferred<void>();
167-
// Create a dummy environment so we can update its executable later. We can't choose a real environment here.
168-
// Executables inside real environments can be symlinks, so writing on them can result in the real executable
169-
// being updated instead of the symlink.
170-
const executable = await workspaceVenvs.createDummyEnv('one');
171-
// Wait before the change event has been sent. If both operations occur almost simultaneously no event is sent.
172-
await sleep(100);
173-
await setupLocator(async (e) => {
174-
actualEvent = e;
175-
deferred.resolve();
176-
});
177-
178-
await workspaceVenvs.update(executable);
179-
await waitForChangeToBeDetected(deferred);
180-
const isFound = await isLocated(executable);
181-
182-
assert.ok(isFound);
183-
// Detecting kind of virtual env depends on the file structure around the executable, so we need to wait before
184-
// attempting to verify it. Omitting that check as we can never deterministically say when it's ready to check.
185-
assert.deepEqual(actualEvent!.type, FileChangeType.Changed, 'Wrong event emitted');
186-
});
11+
testLocatorWatcher(
12+
testWorkspaceFolder,
13+
<locatorFactoryFuncType>createWorkspaceVirtualEnvLocator,
14+
{ arg: testWorkspaceFolder },
15+
);
18716
});

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.venvs/nonvenv/python‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.venvs/nonvenv/python.exe‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.virtualenvs/nonvenv/python‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.virtualenvs/nonvenv/python.exe‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.virtualenvs/nonvenv/python3‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/.virtualenvs/nonvenv/python3.exe‎

Whitespace-only changes.

‎src/test/pythonEnvironments/common/envlayouts/virtualhome/workonhome/nonvenv/python‎

Whitespace-only changes.

0 commit comments

Comments
 (0)