Skip to content

Commit 20a7bd5

Browse files
DonJayamanneDonJayamanne
authored
Pin python packages used in extension and remove them repo (microsoft#3513)
* Move python dependencies into libs * Delete parso, isort and jedi * Deleted experimental folder from pythonFiles * deleted old ptvsd folder * rename requirements file * Updated travis * Install deps using gulp * Use CI python * Fix path to ptvsd * install distro * upgrade in place
1 parent 19d2a73 commit 20a7bd5

147 files changed

Lines changed: 104 additions & 35780 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ coverage/
1616
**/.venv*/
1717
precommit.hook
1818
pythonFiles/experimental/ptvsd/**
19+
pythonFiles/lib/**
1920
debug_coverage*/**
2021
languageServer/**
2122
languageServer.*/**

‎.travis.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ install:
5959
- npx gulp prePublishNonBundle
6060
- npx gulp hygiene-modified
6161
- python -m pip install --upgrade pip
62-
- python -m pip install --upgrade -r requirements.txt
63-
- python -m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd
62+
- python -m pip install --upgrade -r ./build/test-requirements.txt
63+
- npx gulp installPythonLibs
6464

6565
script:
6666
- if [ $UNIT_TEST == "true" ]; then
@@ -73,7 +73,7 @@ script:
7373
fi
7474
- npm run debugger-coverage
7575
- if [ $FUNCTIONAL_TEST == "true" ]; then
76-
python -m pip install --upgrade -r functionalTestRequirements.txt;
76+
python -m pip install --upgrade -r ./build/functional-test-requirements.txt;
7777
npm run test:functional;
7878
fi
7979
- if [ $SINGLE_WORKSPACE_TEST == "true" ]; then

‎.vscodeignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ CODING_STANDARDS.md
1515
CONTRIBUTING.md
1616
CONTRIBUTING - LANGUAGE SERVER.md
1717
coverconfig.json
18-
functionalTestRequirements.txt
1918
gulpfile.js
2019
package.datascience-ui.dependencies.json
2120
package-lock.json
@@ -56,6 +55,9 @@ out/test/**
5655
out/testMultiRootWkspc/**
5756
precommit.hook
5857
pythonFiles/**/*.pyc
58+
pythonFiles/lib/**/*.dist-info/**
59+
pythonFiles/lib/**/*.egg-info/**
60+
pythonFiles/lib/python/bin/**
5961
requirements.txt
6062
scripts/**
6163
src/**

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ cd vscode-python
3737
npm install
3838
python3 -m venv .venv
3939
# Activate the virtual environment as appropriate for your shell.
40-
python -m pip install -r requirements.txt
41-
python -m pip install -t pythonFiles/experimental/ptvsd ptvsd
40+
python3 -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
4241
# Specifying the virtual environment simply varies between shells.
4342
export CI_PYTHON_PATH=`pwd`/.venv/bin/python
4443
```

‎build/ci/templates/compile-and-validate.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,19 +156,19 @@ jobs:
156156

157157

158158
- task: CmdLine@1
159-
displayName: 'pip install requirements'
159+
displayName: 'pip install test requirements'
160160
inputs:
161161
filename: python
162162

163-
arguments: '-m pip install --upgrade -r ./requirements.txt'
163+
arguments: '-m pip install --upgrade -r ./build/test-requirements.txt'
164164

165165

166166
- task: CmdLine@1
167-
displayName: 'pip install ptvsd'
167+
displayName: 'pip install python packages'
168168
inputs:
169169
filename: python
170170

171-
arguments: '-m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd'
171+
arguments: '-m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt'
172172

173173

174174
- task: ArchiveFiles@2

‎build/ci/templates/test-phase.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,15 +149,15 @@ steps:
149149
inputs:
150150
filename: python
151151

152-
arguments: '-m pip install --upgrade -r ./requirements.txt'
152+
arguments: '-m pip install --upgrade -r ./build/test-requirements.txt'
153153

154154

155155
- task: CmdLine@1
156156
displayName: 'pip install ptvsd'
157157
inputs:
158158
filename: python
159159

160-
arguments: '-m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd'
160+
arguments: '-m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt'
161161

162162

163163
- task: Npm@1

functionalTestRequirements.txt renamed to build/functional-test-requirements.txt

File renamed without changes.

‎build/test-requirements.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Install flake8 first, as both flake8 and autopep8 require pycodestyle,
2+
# but flake8 has a tighter pinning.
3+
flake8
4+
autopep8
5+
bandit
6+
black ; python_version>='3.6'
7+
yapf
8+
pylint
9+
pep8
10+
prospector
11+
pydocstyle
12+
nose
13+
pytest==3.6.3
14+
rope
15+
flask
16+
django

‎gulpfile.js‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ const path = require('path');
2222
const jeditor = require("gulp-json-editor");
2323
const del = require('del');
2424
const sourcemaps = require('gulp-sourcemaps');
25-
const fs = require('fs');
25+
const fs = require('fs-extra');
2626
const fsExtra = require('fs-extra');
2727
const remapIstanbul = require('remap-istanbul');
2828
const istanbul = require('istanbul');
@@ -107,8 +107,6 @@ gulp.task('clean:out', () => del(['out']));
107107

108108
gulp.task('clean', gulp.parallel('output:clean', 'cover:clean', 'clean:vsix'));
109109

110-
gulp.task('clean:ptvsd', () => del(['coverage', 'pythonFiles/experimental/ptvsd/*']));
111-
112110
gulp.task('checkNativeDependencies', (done) => {
113111
if (hasNativeDependencies()) {
114112
throw new Error('Native dependencies deteced');
@@ -202,11 +200,32 @@ gulp.task('webpack', async () => {
202200
gulp.task('prePublishBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'clean:cleanExceptTests', 'webpack'));
203201
gulp.task('prePublishNonBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'compile-webviews'));
204202

203+
const installPythonLibArgs = ['-m', 'pip', '--disable-pip-version-check', 'install',
204+
'-t', './pythonFiles/lib/python', '--no-cache-dir', '--implementation', 'py', '--no-deps',
205+
'--upgrade', '-r', 'requirements.txt'];
206+
gulp.task('installPythonLibs', async () => {
207+
const requirements = fs.readFileSync(path.join(__dirname, 'requirements.txt'), 'utf8').split('\n').map(item => item.trim()).filter(item => item.length > 0);
208+
const args = ['-m', 'pip', '--disable-pip-version-check', 'install', '-t', './pythonFiles/lib/python', '--no-cache-dir', '--implementation', 'py', '--no-deps', '--upgrade'];
209+
await Promise.all(requirements.map(async requirement => {
210+
const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args.concat(requirement))
211+
.then(() => true)
212+
.catch(ex => {
213+
console.error('Failed to install Python Libs using \'python3\'', ex);
214+
return false
215+
});
216+
if (!success) {
217+
console.info('Failed to install Python Libs using \'python3\', attempting to install using \'python\'');
218+
await spawnAsync('python', args.concat(requirement))
219+
.catch(ex => console.error('Failed to install Python Libs using \'python\'', ex));
220+
}
221+
}));
222+
});
223+
205224
function spawnAsync(command, args) {
206225
return new Promise((resolve, reject) => {
207226
const proc = spawn(command, args, { cwd: __dirname });
208227
proc.stdout.on('data', data => {
209-
// Log output on CI.
228+
// Log output on CI (else travis times out when there's not output).
210229
if (isCI) {
211230
console.log(data.toString());
212231
}

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1708,7 +1708,6 @@
17081708
"lint-staged": "node gulpfile.js",
17091709
"lint": "tslint src/**/*.ts -t verbose",
17101710
"clean": "gulp clean",
1711-
"clean:ptvsd": "gulp clean:ptvsd",
17121711
"cover:enable": "gulp cover:enable",
17131712
"debugger-coverage": "gulp debugger-coverage",
17141713
"cover:inlinesource": "gulp inlinesource"

0 commit comments

Comments
 (0)