Skip to content

Commit 1b04ed7

Browse files
rchiodorchiodo
authored
Add conda support to nightly flake test (microsoft#10523)
* Add conda environments * Reenable multiple interpreters * Change path for conda environments * Add pip requirements * Get multiple interpreters test to work * Partially working conda search * Get conda working locally * Make windows supported environments * Fix hygiene * Fix memory leak for daemons * Fix hang on conda * Actually retry * Two missing setting change updates * Rework the failure capture * Get rid of memory leak when session dies * Make tests leak less memory * Hygiene problem * More potential cleanup * More changes to clean up memory as soon as possible * Dont auto start jupyter * More leaks found and fixes for breaking tests * Fix missing args when test is shutting down * Put back some of the functional requirements * Update tests to not use static * Fix linter tests * More output for linter tests * Linter only fails with other tests * Install linters in conda too * Make base also have dependencies as it's being picked up anyway * Add news entry * Review feedback * Fix unit tests and one of the functional tests * Functional tests and workspace tests at the same time fail
1 parent 66a9333 commit 1b04ed7

54 files changed

Lines changed: 1029 additions & 703 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.vscode/launch.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,9 @@
240240
"env": {
241241
// Remove `X` prefix to test with real python (for DS functional tests).
242242
"XVSCODE_PYTHON_ROLLING": "1",
243+
// Remove 'X' to turn on all logging in the debug output
244+
"XVSC_PYTHON_FORCE_LOGGING": "1",
243245
// Remove `X` prefix and update path to test with real python interpreter (for DS functional tests).
244-
// Do not use a conda environment (as it needs to be activated and the like).
245246
"XCI_PYTHON_PATH": "<Python Path>"
246247
},
247248
"outFiles": [

‎build/ci/conda_base.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
pandas
2+
jupyter
3+
numpy
4+
matplotlib

‎build/ci/conda_env_1.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name: conda_env_1
2+
dependencies:
3+
- python=3.7
4+
- pandas
5+
- jupyter
6+
- numpy
7+
- matplotlib

‎build/ci/conda_env_2.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name: conda_env_2
2+
dependencies:
3+
- python=3.8
4+
- pandas
5+
- jupyter
6+
- numpy
7+
- matplotlib

‎build/ci/templates/test_phases.yml‎

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -97,16 +97,7 @@ steps:
9797
python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/debugpy/no_wheels --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
9898
displayName: 'pip install system test requirements'
9999
condition: and(succeeded(), eq(variables['NeedsPythonTestReqs'], 'true'))
100-
101-
# Install the additional sqlite requirements
102-
#
103-
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
104-
- bash: |
105-
sudo apt-get install libsqlite3-dev
106-
python -m pip install pysqlite
107-
displayName: 'Setup python to run with sqlite on 2.7'
108-
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), eq(variables['Agent.Os'], 'Linux'), eq(variables['PythonVersion'], '2.7'))
109-
100+
110101
# Install the requirements for functional tests.
111102
#
112103
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
@@ -118,9 +109,69 @@ steps:
118109
- bash: |
119110
python -m pip install numpy
120111
python -m pip install --upgrade -r ./build/functional-test-requirements.txt
112+
python -c "import sys;print(sys.executable)"
121113
displayName: 'pip install functional requirements'
114+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'))
115+
116+
# Add CONDA to the path so anaconda works
117+
#
118+
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
119+
- bash: |
120+
echo "##vso[task.prependpath]$CONDA/bin"
121+
displayName: 'Add conda to the path'
122+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), ne(variables['Agent.Os'], 'Windows_NT'))
123+
124+
# Add CONDA to the path so anaconda works (windows)
125+
#
126+
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
127+
- powershell: |
128+
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
129+
displayName: 'Add conda to the path'
130+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), eq(variables['Agent.Os'], 'Windows_NT'))
131+
132+
# On MAC let CONDA update install paths
133+
#
134+
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
135+
- bash: |
136+
sudo chown -R $USER $CONDA
137+
displayName: 'Give CONDA permission to its own files'
138+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), eq(variables['Agent.Os'], 'Darwin'))
139+
140+
# Create the two anaconda environments
141+
#
142+
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.
143+
#
144+
- script: |
145+
conda env create --quiet --force --file build/ci/conda_env_1.yml
146+
conda env create --quiet --force --file build/ci/conda_env_2.yml
147+
displayName: 'Create CONDA Environments'
122148
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'))
123149
150+
# Run the pip installs in the 3 environments (darwin linux)
151+
- bash: |
152+
source activate base
153+
conda install --quiet -y --file build/ci/conda_base.yml
154+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
155+
source activate conda_env_1
156+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
157+
source activate conda_env_2
158+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
159+
conda deactivate
160+
displayName: 'Install Pip requirements for CONDA envs'
161+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), ne(variables['Agent.Os'], 'Windows_NT'))
162+
163+
# Run the pip installs in the 3 environments (windows)
164+
- script: |
165+
call activate base
166+
conda install --quiet -y --file build/ci/conda_base.yml
167+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
168+
call activate conda_env_1
169+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
170+
call activate conda_env_2
171+
python -m pip install --upgrade -r build/conda-functional-requirements.txt
172+
displayName: 'Install Pip requirements for CONDA envs'
173+
condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'), eq(variables['Agent.Os'], 'Windows_NT'))
174+
124175
# Downgrade pywin32 on Windows due to bug https://github.com/jupyter/notebook/issues/4909
125176
#
126177
# This task will only run if variable `NeedsPythonFunctionalReqs` is true.

‎build/ci/vscode-python-nightly-flake-ci.yaml‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -50,22 +50,6 @@ stages:
5050
steps:
5151
- template: templates/test_phases.yml
5252

53-
- job: 'Py36'
54-
dependsOn: []
55-
timeoutInMinutes: 120
56-
strategy:
57-
matrix:
58-
'Functional':
59-
PythonVersion: '3.6'
60-
TestsToRun: 'testfunctional'
61-
NeedsPythonTestReqs: true
62-
NeedsPythonFunctionalReqs: true
63-
VSCODE_PYTHON_ROLLING: true
64-
pool:
65-
vmImage: 'ubuntu-16.04'
66-
steps:
67-
- template: templates/test_phases.yml
68-
6953
- stage: Mac
7054
dependsOn:
7155
- Build
@@ -85,22 +69,6 @@ stages:
8569
steps:
8670
- template: templates/test_phases.yml
8771

88-
- job: 'Py36'
89-
dependsOn: []
90-
timeoutInMinutes: 120
91-
strategy:
92-
matrix:
93-
'Functional':
94-
PythonVersion: '3.6'
95-
TestsToRun: 'testfunctional'
96-
NeedsPythonTestReqs: true
97-
NeedsPythonFunctionalReqs: true
98-
VSCODE_PYTHON_ROLLING: true
99-
pool:
100-
vmImage: '$(vmImageMacOS)'
101-
steps:
102-
- template: templates/test_phases.yml
103-
10472
- stage: Windows
10573
dependsOn:
10674
- Build
@@ -119,19 +87,3 @@ stages:
11987
vmImage: 'vs2017-win2016'
12088
steps:
12189
- template: templates/test_phases.yml
122-
123-
- job: 'Py36'
124-
dependsOn: []
125-
timeoutInMinutes: 120
126-
strategy:
127-
matrix:
128-
'Functional':
129-
PythonVersion: '3.6'
130-
TestsToRun: 'testfunctional'
131-
NeedsPythonTestReqs: true
132-
NeedsPythonFunctionalReqs: true
133-
VSCODE_PYTHON_ROLLING: true
134-
pool:
135-
vmImage: 'vs2017-win2016'
136-
steps:
137-
- template: templates/test_phases.yml

‎build/conda-functional-requirements.txt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# List of requirements for conda environments that cannot be installed using conda
2+
livelossplot
3+
versioneer
4+
flake8
5+
autopep8
6+
bandit
7+
black ; python_version>='3.6'
8+
yapf
9+
pylint
10+
pycodestyle
11+
prospector
12+
pydocstyle
13+
nose
14+
pytest==4.6.9 # Last version of pytest with Python 2.7 support
15+
rope
16+
flask
17+
django
18+
isort
19+
pathlib2>=2.2.0 ; python_version<'3.6' # Python 2.7 compatibility (pytest)

‎build/functional-test-requirements.txt‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,2 @@
1-
# List of requirements for functional tests
2-
versioneer
3-
jupyter
4-
numpy
5-
matplotlib
6-
pandas
7-
livelossplot
1+
# List of requirements for functional tests
2+
versioneer

‎news/3 Code Health/10134.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add conda environments to nightly test runs

‎src/client/common/asyncDisposableRegistry.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export class AsyncDisposableRegistry implements IAsyncDisposableRegistry {
1212
public async dispose(): Promise<void> {
1313
const promises = this._list.map(l => l.dispose());
1414
await Promise.all(promises);
15+
this._list = [];
1516
}
1617

1718
public push(disposable?: IDisposable | IAsyncDisposable) {

0 commit comments

Comments
 (0)