From 6eab2fe47ed3b36ca78adc1b43754e75ffe7f513 Mon Sep 17 00:00:00 2001 From: David Kutugata Date: Wed, 11 Mar 2020 18:47:14 -0700 Subject: [PATCH 1/5] Davidkutu/port intellisense fix (#10531) * Jupyter autocompletion will only show up on empty lines, instead of appearing in functions. * filter out magic commands instead of ignoring all the jupyter intellisense. * moved the new code to a function and created tests * removed pressCtrlSpace function * added comments * update changelog and delete news file --- CHANGELOG.md | 2 + .../intellisense/intellisenseProvider.ts | 29 ++++++-- .../intellisense.functional.test.tsx | 66 ++++++++++++++++++- src/test/datascience/mockJupyterSession.ts | 2 +- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93279f888468..8e94bbda3067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +1. Jupyter autocompletion will only show magic commands on empty lines, preventing them of appearing in functions. + ([#10023](https://github.com/Microsoft/vscode-python/issues/10023)) 1. Remove extra lines at the end of the file when formatting with Black. ([#1877](https://github.com/Microsoft/vscode-python/issues/1877)) 1. Capitalize `Activate.ps1` in code for PowerShell Core on Linux. diff --git a/src/client/datascience/interactive-common/intellisense/intellisenseProvider.ts b/src/client/datascience/interactive-common/intellisense/intellisenseProvider.ts index 9a4821ca2057..b9882cfa32d7 100644 --- a/src/client/datascience/interactive-common/intellisense/intellisenseProvider.ts +++ b/src/client/datascience/interactive-common/intellisense/intellisenseProvider.ts @@ -36,7 +36,8 @@ import { IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, - INotebook + INotebook, + INotebookCompletion } from '../../types'; import { ICancelIntellisenseRequest, @@ -377,6 +378,7 @@ export class IntellisenseProvider implements IInteractiveWindowListener { request.cellId, cancelSource.token ); + const jupyterCompletions = this.provideJupyterCompletionItems( request.position, request.context, @@ -469,6 +471,8 @@ export class IntellisenseProvider implements IInteractiveWindowListener { const jupyterResults = await activeNotebook.getCompletion(data.text, offsetInCode, cancelToken); if (jupyterResults && jupyterResults.matches) { + const filteredMatches = this.filterJupyterMatches(document, jupyterResults, cellId, position); + const baseOffset = data.offset; const basePosition = document.positionAt(baseOffset); const startPosition = document.positionAt(jupyterResults.cursor.start + baseOffset); @@ -480,11 +484,7 @@ export class IntellisenseProvider implements IInteractiveWindowListener { endColumn: endPosition.character + 1 }; return { - suggestions: convertStringsToSuggestions( - jupyterResults.matches, - range, - jupyterResults.metadata - ), + suggestions: convertStringsToSuggestions(filteredMatches, range, jupyterResults.metadata), incomplete: false }; } @@ -502,6 +502,23 @@ export class IntellisenseProvider implements IInteractiveWindowListener { }; } + // The suggestions that the kernel is giving always include magic commands. That is confusing to the user. + // This function is called by provideJupyterCompletionItems to filter those magic commands when not in an empty line of code. + private filterJupyterMatches( + document: IntellisenseDocument, + jupyterResults: INotebookCompletion, + cellId: string, + position: monacoEditor.Position + ) { + // If the line we're analyzing is empty or a whitespace, we filter out the magic commands + // as its confusing to see them appear after a . or inside (). + const pos = document.convertToDocumentPosition(cellId, position.lineNumber, position.column); + const line = document.lineAt(pos); + return line.isEmptyOrWhitespace + ? jupyterResults.matches + : jupyterResults.matches.filter(match => !match.startsWith('%')); + } + private postTimedResponse( promises: Promise[], message: T, diff --git a/src/test/datascience/intellisense.functional.test.tsx b/src/test/datascience/intellisense.functional.test.tsx index bf4a8843ff4f..c8b781b263bb 100644 --- a/src/test/datascience/intellisense.functional.test.tsx +++ b/src/test/datascience/intellisense.functional.test.tsx @@ -12,7 +12,7 @@ import { MonacoEditor } from '../../datascience-ui/react-common/monacoEditor'; import { noop } from '../core'; import { DataScienceIocContainer } from './dataScienceIocContainer'; import { getOrCreateInteractiveWindow, runMountedTest } from './interactiveWindowTestHelpers'; -import { getInteractiveEditor, typeCode } from './testHelpers'; +import { enterEditorKey, getInteractiveEditor, typeCode } from './testHelpers'; // tslint:disable:max-func-body-length trailing-comma no-any no-multiline-string suite('DataScience Intellisense tests', () => { @@ -69,6 +69,14 @@ suite('DataScience Intellisense tests', () => { assert.ok(innerTexts.includes(expectedSpan), 'Intellisense row not matching'); } + function verifyIntellisenseNotVisible( + wrapper: ReactWrapper, React.Component>, + expectedSpan: string + ) { + const innerTexts = getIntellisenseTextLines(wrapper); + assert.ok(!innerTexts.includes(expectedSpan), 'Intellisense row is showing'); + } + function waitForSuggestion( wrapper: ReactWrapper, React.Component> ): { disposable: IDisposable; promise: Promise } { @@ -230,4 +238,60 @@ suite('DataScience Intellisense tests', () => { return ioc; } ); + + runMountedTest( + 'Filtered Jupyter autocomplete, verify magic commands appear', + async wrapper => { + if (ioc.mockJupyter) { + // This test only works when mocking. + + // Create an interactive window so that it listens to the results. + const interactiveWindow = await getOrCreateInteractiveWindow(ioc); + await interactiveWindow.show(); + + // Then enter some code. Don't submit, we're just testing that autocomplete appears + const suggestion = waitForSuggestion(wrapper); + typeCode(getInteractiveEditor(wrapper), 'print'); + enterEditorKey(wrapper, { code: ' ', ctrlKey: true }); + await suggestion.promise; + suggestion.disposable.dispose(); + verifyIntellisenseNotVisible(wrapper, '%%bash'); + + // Force suggestion box to disappear so that shutdown doesn't try to generate suggestions + // while we're destroying the editor. + clearEditor(wrapper); + } + }, + () => { + return ioc; + } + ); + + runMountedTest( + 'Filtered Jupyter autocomplete, verify magic commands are filtered', + async wrapper => { + if (ioc.mockJupyter) { + // This test only works when mocking. + + // Create an interactive window so that it listens to the results. + const interactiveWindow = await getOrCreateInteractiveWindow(ioc); + await interactiveWindow.show(); + + // Then enter some code. Don't submit, we're just testing that autocomplete appears + const suggestion = waitForSuggestion(wrapper); + typeCode(getInteractiveEditor(wrapper), ' '); + enterEditorKey(wrapper, { code: ' ', ctrlKey: true }); + await suggestion.promise; + suggestion.disposable.dispose(); + verifyIntellisenseVisible(wrapper, '%%bash'); + + // Force suggestion box to disappear so that shutdown doesn't try to generate suggestions + // while we're destroying the editor. + clearEditor(wrapper); + } + }, + () => { + return ioc; + } + ); }); diff --git a/src/test/datascience/mockJupyterSession.ts b/src/test/datascience/mockJupyterSession.ts index 645d5d655098..27ffb2017831 100644 --- a/src/test/datascience/mockJupyterSession.ts +++ b/src/test/datascience/mockJupyterSession.ts @@ -153,7 +153,7 @@ export class MockJupyterSession implements IJupyterSession { return { content: { - matches: ['printly'], // This keeps this in the intellisense when the editor pairs down results + matches: ['printly', '%%bash'], // This keeps this in the intellisense when the editor pairs down results cursor_start: 0, cursor_end: 7, status: 'ok', From f7cf79b1010836253dbe349082614cf32ac8e3b8 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 12 Mar 2020 08:41:12 -0700 Subject: [PATCH 2/5] Rename datascience to datascience_modules (#10525) (#10528) * Rename datascience to datascience_modules (#10525) * Rename datascience to datascience_modules * Change name to something even harder to get wrong * Fix unit test failure for release --- pythonFiles/tests/ipython/scripts.py | 14 +- .../__init__.py | 0 .../daemon/README.md | 2 +- .../daemon/__init__.py | 0 .../daemon/__main__.py | 2 +- .../daemon/daemon_output.py | 0 .../daemon/daemon_python.py | 4 +- .../dummyJupyter.py | 56 ++--- .../getJupyterKernels.py | 0 .../getJupyterKernelspecVersion.py | 0 .../getJupyterVariableDataFrameInfo.py | 236 +++++++++--------- .../getJupyterVariableDataFrameRows.py | 100 ++++---- .../getJupyterVariableList.py | 0 .../getJupyterVariableValue.py | 0 .../getServerInfo.py | 0 .../jupyter_daemon.py | 2 +- .../jupyter_nbInstalled.py | 0 src/client/common/process/pythonDaemonPool.ts | 6 +- src/client/datascience/constants.ts | 2 +- .../jupyter/interpreter/jupyterCommand.ts | 20 +- ...pyterCommandInterpreterExecutionService.ts | 2 +- ...erInterpreterSubCommandExecutionService.ts | 11 +- .../datascience/jupyter/jupyterVariables.ts | 14 +- .../process/pythonDaemon.functional.test.ts | 4 +- .../pythonDaemonPool.functional.test.ts | 6 +- .../process/pythonDaemonPool.unit.test.ts | 4 +- src/test/datascience/execution.unit.test.ts | 28 ++- ...terSubCommandExecutionService.unit.test.ts | 2 +- src/test/datascience/mockJupyterManager.ts | 14 +- 29 files changed, 306 insertions(+), 223 deletions(-) rename pythonFiles/{datascience => vscode_datascience_helpers}/__init__.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/daemon/README.md (92%) rename pythonFiles/{datascience => vscode_datascience_helpers}/daemon/__init__.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/daemon/__main__.py (98%) rename pythonFiles/{datascience => vscode_datascience_helpers}/daemon/daemon_output.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/daemon/daemon_python.py (98%) rename pythonFiles/{datascience => vscode_datascience_helpers}/dummyJupyter.py (96%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterKernels.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterKernelspecVersion.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterVariableDataFrameInfo.py (92%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterVariableDataFrameRows.py (86%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterVariableList.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getJupyterVariableValue.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/getServerInfo.py (100%) rename pythonFiles/{datascience => vscode_datascience_helpers}/jupyter_daemon.py (99%) rename pythonFiles/{datascience => vscode_datascience_helpers}/jupyter_nbInstalled.py (100%) diff --git a/pythonFiles/tests/ipython/scripts.py b/pythonFiles/tests/ipython/scripts.py index 4fe5c8473485..cf65e8480acb 100644 --- a/pythonFiles/tests/ipython/scripts.py +++ b/pythonFiles/tests/ipython/scripts.py @@ -46,7 +46,7 @@ def execute_script(file, replace_dict=dict([])): def get_variables(capsys): path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableList.py") + os.path.join(path, "../../vscode_datascience_helpers/getJupyterVariableList.py") ) if execute_script(file): read_out = capsys.readouterr() @@ -65,7 +65,9 @@ def get_variable_value(variables, name, capsys): varJson = find_variable_json(variables, name) path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableValue.py") + os.path.join( + path, "../../vscode_datascience_helpers/getJupyterVariableValue.py" + ) ) keys = dict([("_VSCode_JupyterTestValue", json.dumps(varJson))]) if execute_script(file, keys): @@ -79,7 +81,9 @@ def get_data_frame_info(variables, name, capsys): varJson = find_variable_json(variables, name) path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableDataFrameInfo.py") + os.path.join( + path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py" + ) ) keys = dict([("_VSCode_JupyterTestValue", json.dumps(varJson))]) if execute_script(file, keys): @@ -92,7 +96,9 @@ def get_data_frame_info(variables, name, capsys): def get_data_frame_rows(varJson, start, end, capsys): path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableDataFrameRows.py") + os.path.join( + path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameRows.py" + ) ) keys = dict( [ diff --git a/pythonFiles/datascience/__init__.py b/pythonFiles/vscode_datascience_helpers/__init__.py similarity index 100% rename from pythonFiles/datascience/__init__.py rename to pythonFiles/vscode_datascience_helpers/__init__.py diff --git a/pythonFiles/datascience/daemon/README.md b/pythonFiles/vscode_datascience_helpers/daemon/README.md similarity index 92% rename from pythonFiles/datascience/daemon/README.md rename to pythonFiles/vscode_datascience_helpers/daemon/README.md index 45b4ce6a97c0..8fdbc5eabdde 100644 --- a/pythonFiles/datascience/daemon/README.md +++ b/pythonFiles/vscode_datascience_helpers/daemon/README.md @@ -7,7 +7,7 @@ const env = { PYTHONUNBUFFERED: '1', PYTHONPATH: '/pythonFiles:/pythonFiles/lib/python' } -const childProcess = cp.spawn('', ['-m', 'datascience.daemon', '-v', '--log-file=log.log'], {env}); +const childProcess = cp.spawn('', ['-m', 'vscode_datascience_helpers.daemon', '-v', '--log-file=log.log'], {env}); const connection = rpc.createMessageConnection(new rpc.StreamMessageReader(childProcess.stdout),new rpc.StreamMessageWriter(childProcess.stdin)); connection.onClose(() => console.error('Closed')); diff --git a/pythonFiles/datascience/daemon/__init__.py b/pythonFiles/vscode_datascience_helpers/daemon/__init__.py similarity index 100% rename from pythonFiles/datascience/daemon/__init__.py rename to pythonFiles/vscode_datascience_helpers/daemon/__init__.py diff --git a/pythonFiles/datascience/daemon/__main__.py b/pythonFiles/vscode_datascience_helpers/daemon/__main__.py similarity index 98% rename from pythonFiles/datascience/daemon/__main__.py rename to pythonFiles/vscode_datascience_helpers/daemon/__main__.py index c1c047f233b9..2d03e03e613a 100644 --- a/pythonFiles/datascience/daemon/__main__.py +++ b/pythonFiles/vscode_datascience_helpers/daemon/__main__.py @@ -20,7 +20,7 @@ def add_arguments(parser): parser.add_argument( "--daemon-module", - default="datascience.daemon.daemon_python", + default="vscode_datascience_helpers.daemon.daemon_python", help="Daemon Module", ) diff --git a/pythonFiles/datascience/daemon/daemon_output.py b/pythonFiles/vscode_datascience_helpers/daemon/daemon_output.py similarity index 100% rename from pythonFiles/datascience/daemon/daemon_output.py rename to pythonFiles/vscode_datascience_helpers/daemon/daemon_output.py diff --git a/pythonFiles/datascience/daemon/daemon_python.py b/pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py similarity index 98% rename from pythonFiles/datascience/daemon/daemon_python.py rename to pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py index d9d85c587123..694934fefdb8 100644 --- a/pythonFiles/datascience/daemon/daemon_python.py +++ b/pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py @@ -8,7 +8,7 @@ import traceback import runpy import importlib -from datascience.daemon.daemon_output import ( +from vscode_datascience_helpers.daemon.daemon_output import ( CustomWriter, IORedirector, get_io_buffers, @@ -63,7 +63,7 @@ def _decorator(self, *args, **kwargs): class PythonDaemon(MethodDispatcher): """ Base Python Daemon with simple methods to check if a module exists, get version info and the like. - To add additional methods, please create a separate class based off this and pass in the arg `--daemon-module` to `datascience.daemon`. + To add additional methods, please create a separate class based off this and pass in the arg `--daemon-module` to `vscode_datascience_helpers.daemon`. """ def __init__(self, rx, tx): diff --git a/pythonFiles/datascience/dummyJupyter.py b/pythonFiles/vscode_datascience_helpers/dummyJupyter.py similarity index 96% rename from pythonFiles/datascience/dummyJupyter.py rename to pythonFiles/vscode_datascience_helpers/dummyJupyter.py index 211af8cec79d..1a1a6cc07cb9 100644 --- a/pythonFiles/datascience/dummyJupyter.py +++ b/pythonFiles/vscode_datascience_helpers/dummyJupyter.py @@ -1,28 +1,28 @@ -# This file can mimic juypter running. Useful for testing jupyter crash handling - -import sys -import argparse -import time - - -def main(): - print("hello from dummy jupyter") - parser = argparse.ArgumentParser() - parser.add_argument("--version", type=bool, default=False, const=True, nargs="?") - parser.add_argument("notebook", type=bool, default=False, const=True, nargs="?") - parser.add_argument("--no-browser", type=bool, default=False, const=True, nargs="?") - parser.add_argument("--notebook-dir", default="") - parser.add_argument("--config", default="") - results = parser.parse_args() - if results.version: - print("1.1.dummy") - else: - print( - "http://localhost:8888/?token=012f08663a68e279fe0a5335e0b5dfe44759ddcccf0b3a56" - ) - time.sleep(5) - raise Exception("Dummy is dead") - - -if __name__ == "__main__": - main() +# This file can mimic juypter running. Useful for testing jupyter crash handling + +import sys +import argparse +import time + + +def main(): + print("hello from dummy jupyter") + parser = argparse.ArgumentParser() + parser.add_argument("--version", type=bool, default=False, const=True, nargs="?") + parser.add_argument("notebook", type=bool, default=False, const=True, nargs="?") + parser.add_argument("--no-browser", type=bool, default=False, const=True, nargs="?") + parser.add_argument("--notebook-dir", default="") + parser.add_argument("--config", default="") + results = parser.parse_args() + if results.version: + print("1.1.dummy") + else: + print( + "http://localhost:8888/?token=012f08663a68e279fe0a5335e0b5dfe44759ddcccf0b3a56" + ) + time.sleep(5) + raise Exception("Dummy is dead") + + +if __name__ == "__main__": + main() diff --git a/pythonFiles/datascience/getJupyterKernels.py b/pythonFiles/vscode_datascience_helpers/getJupyterKernels.py similarity index 100% rename from pythonFiles/datascience/getJupyterKernels.py rename to pythonFiles/vscode_datascience_helpers/getJupyterKernels.py diff --git a/pythonFiles/datascience/getJupyterKernelspecVersion.py b/pythonFiles/vscode_datascience_helpers/getJupyterKernelspecVersion.py similarity index 100% rename from pythonFiles/datascience/getJupyterKernelspecVersion.py rename to pythonFiles/vscode_datascience_helpers/getJupyterKernelspecVersion.py diff --git a/pythonFiles/datascience/getJupyterVariableDataFrameInfo.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py similarity index 92% rename from pythonFiles/datascience/getJupyterVariableDataFrameInfo.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py index 6a2a2bba9a0f..856f5e282a18 100644 --- a/pythonFiles/datascience/getJupyterVariableDataFrameInfo.py +++ b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py @@ -1,116 +1,120 @@ -# Query Jupyter server for the info about a dataframe -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json - -# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types -_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']" - -# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable -# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable -_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") - -# Function to compute row count for a value -def _VSCODE_getRowCount(var): - if hasattr(var, "shape"): - try: - # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it - if isinstance(var.shape, tuple): - return var.shape[0] - except TypeError: - return 0 - elif hasattr(var, "__len__"): - try: - return len(var) - except TypeError: - return 0 - - -# First check to see if we are a supported type, this prevents us from adding types that are not supported -# and also keeps our types in sync with what the variable explorer says that we support -if _VSCODE_targetVariable["type"] not in _VSCode_supportsDataExplorer: - del _VSCode_supportsDataExplorer - print(_VSCODE_json.dumps(_VSCODE_targetVariable)) - del _VSCODE_targetVariable -else: - del _VSCode_supportsDataExplorer - _VSCODE_evalResult = eval(_VSCODE_targetVariable["name"]) - - # Figure out shape if not already there. Use the shape to compute the row count - _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_evalResult) - - # Turn the eval result into a df - _VSCODE_df = _VSCODE_evalResult - if isinstance(_VSCODE_evalResult, list): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) - elif isinstance(_VSCODE_evalResult, dict): - _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) - elif _VSCODE_targetVariable["type"] == "ndarray": - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - elif hasattr(_VSCODE_df, "toPandas"): - _VSCODE_df = _VSCODE_df.toPandas() - _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_df) - - # If any rows, use pandas json to convert a single row to json. Extract - # the column names and types from the json so we match what we'll fetch when - # we ask for all of the rows - if ( - hasattr(_VSCODE_targetVariable, "rowCount") - and _VSCODE_targetVariable["rowCount"] - ): - try: - _VSCODE_row = _VSCODE_df.iloc[0:1] - _VSCODE_json_row = _VSCODE_pd_json.to_json( - None, _VSCODE_row, date_format="iso" - ) - _VSCODE_columnNames = list(_VSCODE_json.loads(_VSCODE_json_row)) - del _VSCODE_row - del _VSCODE_json_row - except: - _VSCODE_columnNames = list(_VSCODE_df) - else: - _VSCODE_columnNames = list(_VSCODE_df) - - # Compute the index column. It may have been renamed - _VSCODE_indexColumn = _VSCODE_df.index.name if _VSCODE_df.index.name else "index" - _VSCODE_columnTypes = list(_VSCODE_df.dtypes) - del _VSCODE_df - - # Make sure the index column exists - if _VSCODE_indexColumn not in _VSCODE_columnNames: - _VSCODE_columnNames.insert(0, _VSCODE_indexColumn) - _VSCODE_columnTypes.insert(0, "int64") - - # Then loop and generate our output json - _VSCODE_columns = [] - for _VSCODE_n in range(0, len(_VSCODE_columnNames)): - _VSCODE_column_type = _VSCODE_columnTypes[_VSCODE_n] - _VSCODE_column_name = str(_VSCODE_columnNames[_VSCODE_n]) - _VSCODE_colobj = {} - _VSCODE_colobj["key"] = _VSCODE_column_name - _VSCODE_colobj["name"] = _VSCODE_column_name - _VSCODE_colobj["type"] = str(_VSCODE_column_type) - _VSCODE_columns.append(_VSCODE_colobj) - del _VSCODE_column_name - del _VSCODE_column_type - - del _VSCODE_columnNames - del _VSCODE_columnTypes - - # Save this in our target - _VSCODE_targetVariable["columns"] = _VSCODE_columns - _VSCODE_targetVariable["indexColumn"] = _VSCODE_indexColumn - del _VSCODE_columns - del _VSCODE_indexColumn - - # Transform this back into a string - print(_VSCODE_json.dumps(_VSCODE_targetVariable)) - del _VSCODE_targetVariable - - # Cleanup imports - del _VSCODE_json - del _VSCODE_pd - del _VSCODE_pd_json +# Query Jupyter server for the info about a dataframe +import json as _VSCODE_json +import pandas as _VSCODE_pd +import pandas.io.json as _VSCODE_pd_json +import builtins as _VSCODE_builtins + +# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types +_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']" + +# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable +# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable +_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") + +# Function to compute row count for a value +def _VSCODE_getRowCount(var): + if hasattr(var, "shape"): + try: + # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it + if isinstance(var.shape, tuple): + return var.shape[0] + except TypeError: + return 0 + elif hasattr(var, "__len__"): + try: + return _VSCODE_builtins.len(var) + except TypeError: + return 0 + + +# First check to see if we are a supported type, this prevents us from adding types that are not supported +# and also keeps our types in sync with what the variable explorer says that we support +if _VSCODE_targetVariable["type"] not in _VSCode_supportsDataExplorer: + del _VSCode_supportsDataExplorer + print(_VSCODE_json.dumps(_VSCODE_targetVariable)) + del _VSCODE_targetVariable +else: + del _VSCode_supportsDataExplorer + _VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) + + # Figure out shape if not already there. Use the shape to compute the row count + _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_evalResult) + + # Turn the eval result into a df + _VSCODE_df = _VSCODE_evalResult + if isinstance(_VSCODE_evalResult, list): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) + elif isinstance(_VSCODE_evalResult, dict): + _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) + elif _VSCODE_targetVariable["type"] == "ndarray": + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + elif hasattr(_VSCODE_df, "toPandas"): + _VSCODE_df = _VSCODE_df.toPandas() + _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_df) + + # If any rows, use pandas json to convert a single row to json. Extract + # the column names and types from the json so we match what we'll fetch when + # we ask for all of the rows + if ( + hasattr(_VSCODE_targetVariable, "rowCount") + and _VSCODE_targetVariable["rowCount"] + ): + try: + _VSCODE_row = _VSCODE_df.iloc[0:1] + _VSCODE_json_row = _VSCODE_pd_json.to_json( + None, _VSCODE_row, date_format="iso" + ) + _VSCODE_columnNames = list(_VSCODE_json.loads(_VSCODE_json_row)) + del _VSCODE_row + del _VSCODE_json_row + except: + _VSCODE_columnNames = list(_VSCODE_df) + else: + _VSCODE_columnNames = list(_VSCODE_df) + + # Compute the index column. It may have been renamed + _VSCODE_indexColumn = _VSCODE_df.index.name if _VSCODE_df.index.name else "index" + _VSCODE_columnTypes = _VSCODE_builtins.list(_VSCODE_df.dtypes) + del _VSCODE_df + + # Make sure the index column exists + if _VSCODE_indexColumn not in _VSCODE_columnNames: + _VSCODE_columnNames.insert(0, _VSCODE_indexColumn) + _VSCODE_columnTypes.insert(0, "int64") + + # Then loop and generate our output json + _VSCODE_columns = [] + for _VSCODE_n in _VSCODE_builtins.range( + 0, _VSCODE_builtins.len(_VSCODE_columnNames) + ): + _VSCODE_column_type = _VSCODE_columnTypes[_VSCODE_n] + _VSCODE_column_name = str(_VSCODE_columnNames[_VSCODE_n]) + _VSCODE_colobj = {} + _VSCODE_colobj["key"] = _VSCODE_column_name + _VSCODE_colobj["name"] = _VSCODE_column_name + _VSCODE_colobj["type"] = str(_VSCODE_column_type) + _VSCODE_columns.append(_VSCODE_colobj) + del _VSCODE_column_name + del _VSCODE_column_type + + del _VSCODE_columnNames + del _VSCODE_columnTypes + + # Save this in our target + _VSCODE_targetVariable["columns"] = _VSCODE_columns + _VSCODE_targetVariable["indexColumn"] = _VSCODE_indexColumn + del _VSCODE_columns + del _VSCODE_indexColumn + + # Transform this back into a string + print(_VSCODE_json.dumps(_VSCODE_targetVariable)) + del _VSCODE_targetVariable + + # Cleanup imports + del _VSCODE_json + del _VSCODE_pd + del _VSCODE_pd_json + del _VSCODE_builtins diff --git a/pythonFiles/datascience/getJupyterVariableDataFrameRows.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py similarity index 86% rename from pythonFiles/datascience/getJupyterVariableDataFrameRows.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py index 697cc14ad1b6..e7d73ae656b2 100644 --- a/pythonFiles/datascience/getJupyterVariableDataFrameRows.py +++ b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py @@ -1,48 +1,52 @@ -# Query Jupyter server for the rows of a data frame -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json - -# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable -# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable -_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") -_VSCODE_evalResult = eval(_VSCODE_targetVariable["name"]) - -# _VSCode_JupyterStartRow and _VSCode_JupyterEndRow should be replaced dynamically with the literals -# for our start and end rows -_VSCODE_startRow = max(_VSCode_JupyterStartRow, 0) -_VSCODE_endRow = min(_VSCode_JupyterEndRow, _VSCODE_targetVariable["rowCount"]) - -# Assume we have a dataframe. If not, turn our eval result into a dataframe -_VSCODE_df = _VSCODE_evalResult -if isinstance(_VSCODE_evalResult, list): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) -elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) -elif isinstance(_VSCODE_evalResult, dict): - _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) -elif _VSCODE_targetVariable["type"] == "ndarray": - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) -elif hasattr(_VSCODE_df, "toPandas"): - _VSCODE_df = _VSCODE_df.toPandas() -# If not a known type, then just let pandas handle it. -elif not (hasattr(_VSCODE_df, "iloc")): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - -# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON -_VSCODE_rows = _VSCODE_df.iloc[_VSCODE_startRow:_VSCODE_endRow] -_VSCODE_result = _VSCODE_pd_json.to_json( - None, _VSCODE_rows, orient="table", date_format="iso" -) -print(_VSCODE_result) - -# Cleanup our variables -del _VSCODE_df -del _VSCODE_endRow -del _VSCODE_startRow -del _VSCODE_rows -del _VSCODE_result -del _VSCODE_json -del _VSCODE_pd -del _VSCODE_pd_json +# Query Jupyter server for the rows of a data frame +import json as _VSCODE_json +import pandas as _VSCODE_pd +import pandas.io.json as _VSCODE_pd_json +import builtins as _VSCODE_builtins + +# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable +# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable +_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") +_VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) + +# _VSCode_JupyterStartRow and _VSCode_JupyterEndRow should be replaced dynamically with the literals +# for our start and end rows +_VSCODE_startRow = _VSCODE_builtins.max(_VSCode_JupyterStartRow, 0) +_VSCODE_endRow = _VSCODE_builtins.min( + _VSCode_JupyterEndRow, _VSCODE_targetVariable["rowCount"] +) + +# Assume we have a dataframe. If not, turn our eval result into a dataframe +_VSCODE_df = _VSCODE_evalResult +if isinstance(_VSCODE_evalResult, list): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) +elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) +elif isinstance(_VSCODE_evalResult, dict): + _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) +elif _VSCODE_targetVariable["type"] == "ndarray": + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) +elif hasattr(_VSCODE_df, "toPandas"): + _VSCODE_df = _VSCODE_df.toPandas() +# If not a known type, then just let pandas handle it. +elif not (hasattr(_VSCODE_df, "iloc")): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + +# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON +_VSCODE_rows = _VSCODE_df.iloc[_VSCODE_startRow:_VSCODE_endRow] +_VSCODE_result = _VSCODE_pd_json.to_json( + None, _VSCODE_rows, orient="table", date_format="iso" +) +print(_VSCODE_result) + +# Cleanup our variables +del _VSCODE_df +del _VSCODE_endRow +del _VSCODE_startRow +del _VSCODE_rows +del _VSCODE_result +del _VSCODE_json +del _VSCODE_pd +del _VSCODE_pd_json +del _VSCODE_builtins diff --git a/pythonFiles/datascience/getJupyterVariableList.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableList.py similarity index 100% rename from pythonFiles/datascience/getJupyterVariableList.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableList.py diff --git a/pythonFiles/datascience/getJupyterVariableValue.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableValue.py similarity index 100% rename from pythonFiles/datascience/getJupyterVariableValue.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableValue.py diff --git a/pythonFiles/datascience/getServerInfo.py b/pythonFiles/vscode_datascience_helpers/getServerInfo.py similarity index 100% rename from pythonFiles/datascience/getServerInfo.py rename to pythonFiles/vscode_datascience_helpers/getServerInfo.py diff --git a/pythonFiles/datascience/jupyter_daemon.py b/pythonFiles/vscode_datascience_helpers/jupyter_daemon.py similarity index 99% rename from pythonFiles/datascience/jupyter_daemon.py rename to pythonFiles/vscode_datascience_helpers/jupyter_daemon.py index be51abab85b8..f83224a48d70 100644 --- a/pythonFiles/datascience/jupyter_daemon.py +++ b/pythonFiles/vscode_datascience_helpers/jupyter_daemon.py @@ -6,7 +6,7 @@ import os import subprocess import sys -from datascience.daemon.daemon_python import ( +from vscode_datascience_helpers.daemon.daemon_python import ( error_decorator, PythonDaemon as BasePythonDaemon, change_exec_context, diff --git a/pythonFiles/datascience/jupyter_nbInstalled.py b/pythonFiles/vscode_datascience_helpers/jupyter_nbInstalled.py similarity index 100% rename from pythonFiles/datascience/jupyter_nbInstalled.py rename to pythonFiles/vscode_datascience_helpers/jupyter_nbInstalled.py diff --git a/src/client/common/process/pythonDaemonPool.ts b/src/client/common/process/pythonDaemonPool.ts index 1248a0d8756a..7189f7c58d96 100644 --- a/src/client/common/process/pythonDaemonPool.ts +++ b/src/client/common/process/pythonDaemonPool.ts @@ -144,7 +144,11 @@ export class PythonDaemonExecutionServicePool implements IPythonDaemonExecutionS loggingArgs ); const env = this.envVariables; - const daemonProc = this.pythonExecutionService!.execModuleObservable('datascience.daemon', args, { env }); + const daemonProc = this.pythonExecutionService!.execModuleObservable( + 'vscode_datascience_helpers.daemon', + args, + { env } + ); if (!daemonProc.proc) { throw new Error('Failed to create Daemon Proc'); } diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 3ec1e4f4e14d..16e7a7f388fd 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -12,7 +12,7 @@ export const DefaultTheme = 'Default Light+'; export const JUPYTER_OUTPUT_CHANNEL = 'JUPYTER_OUTPUT_CHANNEL'; // Python Module to be used when instantiating the Python Daemon. -export const PythonDaemonModule = 'datascience.jupyter_daemon'; +export const PythonDaemonModule = 'vscode_datascience_helpers.jupyter_daemon'; // List of 'language' names that we know about. All should be lower case as that's how we compare. export const KnownNotebookLanguages: string[] = [ diff --git a/src/client/datascience/jupyter/interpreter/jupyterCommand.ts b/src/client/datascience/jupyter/interpreter/jupyterCommand.ts index 49e5a970bd82..342da28034a4 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterCommand.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterCommand.ts @@ -107,7 +107,14 @@ class InterpreterJupyterCommand implements IJupyterCommand { ) { try { const output = await svc.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'jupyter_nbInstalled.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'jupyter_nbInstalled.py' + ) + ], {} ); if (output.stdout.toLowerCase().includes('available')) { @@ -264,7 +271,7 @@ export class InterpreterJupyterKernelSpecCommand extends InterpreterJupyterComma bypassCondaExecution: true }); return activatedEnv.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernels.py')], + [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getJupyterKernels.py')], { ...options, throwOnStdErr: true } ); } @@ -275,7 +282,14 @@ export class InterpreterJupyterKernelSpecCommand extends InterpreterJupyterComma bypassCondaExecution: true }); return activatedEnv.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernelspecVersion.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterKernelspecVersion.py' + ) + ], { ...options, throwOnStdErr: true } ); } diff --git a/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts b/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts index 4888ab9c508a..cf9e70ec7a54 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts @@ -102,7 +102,7 @@ export class JupyterCommandFinderInterpreterExecutionService implements IJupyter // We have a small python file here that we will execute to get the server info from all running Jupyter instances const newOptions: SpawnOptions = { mergeStdOutErr: true, token: token }; - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const serverInfoString = await daemon.exec([file], newOptions); let serverInfos: JupyterServerInfo[]; diff --git a/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts b/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts index e91889297535..350ddab38a5c 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts @@ -126,7 +126,7 @@ export class JupyterInterpreterSubCommandExecutionService // We have a small python file here that we will execute to get the server info from all running Jupyter instances const newOptions: SpawnOptions = { mergeStdOutErr: true, token: token }; - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const serverInfoString = await daemon.exec([file], newOptions); let serverInfos: JupyterServerInfo[]; @@ -199,7 +199,14 @@ export class JupyterInterpreterSubCommandExecutionService // Possible we cannot import ipykernel for some reason. (use as backup option). const stdoutFromFileExecPromise = daemon .exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernels.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterKernels.py' + ) + ], spawnOptions ) .then(output => output.stdout) diff --git a/src/client/datascience/jupyter/jupyterVariables.ts b/src/client/datascience/jupyter/jupyterVariables.ts index 35620498a1a9..0cba8b381613 100644 --- a/src/client/datascience/jupyter/jupyterVariables.ts +++ b/src/client/datascience/jupyter/jupyterVariables.ts @@ -94,10 +94,20 @@ export class JupyterVariables implements IJupyterVariables { // Private methods // Load our python files for fetching variables private async loadVariableFiles(): Promise { - let file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableDataFrameInfo.py'); + let file = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterVariableDataFrameInfo.py' + ); this.fetchDataFrameInfoScript = await this.fileSystem.readFile(file); - file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableDataFrameRows.py'); + file = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterVariableDataFrameRows.py' + ); this.fetchDataFrameRowsScript = await this.fileSystem.readFile(file); this.filesLoaded = true; diff --git a/src/test/common/process/pythonDaemon.functional.test.ts b/src/test/common/process/pythonDaemon.functional.test.ts index aec0af00fad4..8f81f83d7452 100644 --- a/src/test/common/process/pythonDaemon.functional.test.ts +++ b/src/test/common/process/pythonDaemon.functional.test.ts @@ -59,8 +59,8 @@ suite('Daemon', () => { return this.skip(); } // Enable the following to log everything going on at pyton end. - // pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon', '-v', `--log-file=${path.join(EXTENSION_ROOT_DIR, 'test.log')}`], { env }); - pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon'], { env }); + // pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon', '-v', `--log-file=${path.join(EXTENSION_ROOT_DIR, 'test.log')}`], { env }); + pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon'], { env }); connection = createMessageConnection( new StreamMessageReader(pythonProc.stdout), new StreamMessageWriter(pythonProc.stdin) diff --git a/src/test/common/process/pythonDaemonPool.functional.test.ts b/src/test/common/process/pythonDaemonPool.functional.test.ts index 3796e78c83c4..e8ad27f42a30 100644 --- a/src/test/common/process/pythonDaemonPool.functional.test.ts +++ b/src/test/common/process/pythonDaemonPool.functional.test.ts @@ -75,8 +75,10 @@ suite('Daemon - Python Daemon Pool', () => { logger = mock(ProcessLogger); createDaemonServicesSpy = sinon.spy(DaemonPool.prototype, 'createDaemonServices'); pythonExecutionService = mock(PythonExecutionService); - when(pythonExecutionService.execModuleObservable('datascience.daemon', anything(), anything())).thenCall(() => { - const pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon'], { env }); + when( + pythonExecutionService.execModuleObservable('vscode_datascience_helpers.daemon', anything(), anything()) + ).thenCall(() => { + const pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon'], { env }); const connection = createMessageConnection( new StreamMessageReader(pythonProc.stdout), new StreamMessageWriter(pythonProc.stdin) diff --git a/src/test/common/process/pythonDaemonPool.unit.test.ts b/src/test/common/process/pythonDaemonPool.unit.test.ts index 7b8defe65924..4264220ed352 100644 --- a/src/test/common/process/pythonDaemonPool.unit.test.ts +++ b/src/test/common/process/pythonDaemonPool.unit.test.ts @@ -69,7 +69,9 @@ suite('Daemon - Python Daemon Pool', () => { daemonProc.stdout = new EventEmitter() as any; daemonProc.stderr = new EventEmitter() as any; - when(pythonExecService.execModuleObservable('datascience.daemon', anything(), anything())).thenReturn({ + when( + pythonExecService.execModuleObservable('vscode_datascience_helpers.daemon', anything(), anything()) + ).thenReturn({ proc: daemonProc, dispose: noop, out: undefined as any diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 4ecbe743fb4e..e9f8fd4f2977 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -463,7 +463,12 @@ suite('Jupyter Execution', async () => { return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupPythonService( service, undefined, @@ -502,7 +507,12 @@ suite('Jupyter Execution', async () => { ['kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }) ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupPythonService( service, undefined, @@ -599,7 +609,12 @@ suite('Jupyter Execution', async () => { return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupProcessServiceExec( service, workingPython.path, @@ -638,7 +653,12 @@ suite('Jupyter Execution', async () => { ['-m', 'jupyter', 'kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }) ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupProcessServiceExec( service, missingKernelPython.path, diff --git a/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts b/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts index 45ee5b4d54da..2cf83a1f143b 100644 --- a/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts +++ b/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts @@ -318,7 +318,7 @@ suite('Data Science - Jupyter InterpreterSubCommandExecutionService', () => { assert.equal(output, convertOutput); }); test('Return list of running jupyter servers', async () => { - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const expectedServers: JupyterServerInfo[] = [ { base_url: '1', diff --git a/src/test/datascience/mockJupyterManager.ts b/src/test/datascience/mockJupyterManager.ts index 76b1622667c9..e8ea67321d98 100644 --- a/src/test/datascience/mockJupyterManager.ts +++ b/src/test/datascience/mockJupyterManager.ts @@ -762,7 +762,12 @@ export class MockJupyterManager implements IJupyterSessionManager { }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); this.setupProcessServiceExec(this.processService, workingPython.path, [getServerInfoPath], () => Promise.resolve({ stdout: 'failure to get server infos' }) ); @@ -812,7 +817,12 @@ export class MockJupyterManager implements IJupyterSessionManager { return Promise.resolve({ stdout: JSON.stringify(createKernelSpecs(kernels)) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); this.setupProcessServiceExec(this.processService, workingPython.path, [getServerInfoPath], () => Promise.resolve({ stdout: 'failure to get server infos' }) ); From 31cd1f24f6ee0fb26fec01b17ec9c78edecb3fba Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Wed, 18 Mar 2020 14:52:33 -0700 Subject: [PATCH 3/5] Ensure local host only if connection not available (#10600) (#10644) * Ensure local host only if connection not available * Add news item --- news/2 Fixes/10597.md | 1 + .../configuration/resolvers/attach.ts | 5 ++-- src/client/debugger/types.ts | 7 +++++ .../resolvers/attach.unit.test.ts | 26 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 news/2 Fixes/10597.md diff --git a/news/2 Fixes/10597.md b/news/2 Fixes/10597.md new file mode 100644 index 000000000000..c666d8d8d5b4 --- /dev/null +++ b/news/2 Fixes/10597.md @@ -0,0 +1 @@ +Ensure default `host` is not set, if `connect` or `listen` settings are available. diff --git a/src/client/debugger/extension/configuration/resolvers/attach.ts b/src/client/debugger/extension/configuration/resolvers/attach.ts index 0845781dff74..bcfdd4734e79 100644 --- a/src/client/debugger/extension/configuration/resolvers/attach.ts +++ b/src/client/debugger/extension/configuration/resolvers/attach.ts @@ -55,7 +55,8 @@ export class AttachConfigurationResolver extends BaseConfigurationResolver { .deep.equal(debugOptionsAvailable); expect(debugConfig).to.have.property('host', 'localhost'); }); + test('Default host should not be added if connect is available.', async () => { + const pythonFile = 'xyz.py'; + + setupActiveEditor(pythonFile, PYTHON_LANGUAGE); + setupWorkspaces([]); + + const debugConfig = await debugProvider.resolveDebugConfiguration!(undefined, { + request: 'attach', + connect: { host: 'localhost', port: 5678 } + } as AttachRequestArguments); + + expect(debugConfig).to.not.have.property('host', 'localhost'); + }); + test('Default host should not be added if listen is available.', async () => { + const pythonFile = 'xyz.py'; + + setupActiveEditor(pythonFile, PYTHON_LANGUAGE); + setupWorkspaces([]); + + const debugConfig = await debugProvider.resolveDebugConfiguration!(undefined, { + request: 'attach', + listen: { host: 'localhost', port: 5678 } + } as AttachRequestArguments); + + expect(debugConfig).to.not.have.property('host', 'localhost'); + }); test("Ensure 'localRoot' is left unaltered", async () => { const activeFile = 'xyz.py'; const workspaceFolder = createMoqWorkspaceFolder(__dirname); From 9005608baa02b539eba921005d887ff8627539f3 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Wed, 18 Mar 2020 19:39:48 -0700 Subject: [PATCH 4/5] Fix #10437: Update launch.json handling to support "listen" and "connect" (#10517) (#10654) Co-authored-by: Pavel Minaev --- news/1 Enhancements/10437.md | 1 + package.json | 39 +++++++- .../debugger/extension/adapter/factory.ts | 88 +++++++++++-------- .../extension/adapter/factory.unit.test.ts | 43 ++++++--- 4 files changed, 118 insertions(+), 53 deletions(-) create mode 100644 news/1 Enhancements/10437.md diff --git a/news/1 Enhancements/10437.md b/news/1 Enhancements/10437.md new file mode 100644 index 000000000000..169e1a1fdb35 --- /dev/null +++ b/news/1 Enhancements/10437.md @@ -0,0 +1 @@ +Support reverse connection ("listen" in launch.json) from debug adapter to VSCode. diff --git a/package.json b/package.json index 4e5e760cf614..74591c67f73f 100644 --- a/package.json +++ b/package.json @@ -1363,15 +1363,46 @@ }, "attach": { "properties": { + "connect": { + "type": "object", + "label": "Attach by connecting to debugpy over a socket.", + "properties": { + "port": { + "type": "number", + "description": "Port to connect to." + }, + "host": { + "type": "string", + "description": "Hostname or IP address to connect to.", + "default": "127.0.0.1" + } + }, + "required": ["port"] + }, + "listen": { + "type": "object", + "label": "Attach by listening for incoming socket connection from debugpy", + "properties": { + "port": { + "type": "number", + "description": "Port to listen on." + }, + "host": { + "type": "string", + "description": "Hostname or IP address of the interface to listen on.", + "default": "127.0.0.1" + } + }, + "required": ["port"] + }, "port": { "type": "number", - "description": "Debug port to attach", - "default": 0 + "description": "Port to connect to." }, "host": { "type": "string", - "description": "IP Address of the of remote server (default is localhost or use 127.0.0.1).", - "default": "localhost" + "description": "Hostname or IP address to connect to.", + "default": "127.0.0.1" }, "pathMappings": { "type": "array", diff --git a/src/client/debugger/extension/adapter/factory.ts b/src/client/debugger/extension/adapter/factory.ts index 593e19a8a2a9..a612f71f63d1 100644 --- a/src/client/debugger/extension/adapter/factory.ts +++ b/src/client/debugger/extension/adapter/factory.ts @@ -38,48 +38,60 @@ export class DebugAdapterDescriptorFactory implements IDebugAdapterDescriptorFac const configuration = session.configuration as LaunchRequestArguments | AttachRequestArguments; if (this.experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)) { - const isAttach = configuration.request === 'attach'; - const port = configuration.port ?? 0; - // When processId is provided we may have to inject the debugger into the process. - // This is done by the debug adapter, so we need to start it. The adapter will handle injecting the debugger when it receives the attach request. - const processId = configuration.processId ?? 0; - - if (isAttach && processId === 0) { - if (port === 0) { - throw new Error('Port or processId must be specified for request type attach'); - } else { - return new DebugAdapterServer(port, configuration.host); + // There are four distinct scenarios here: + // + // 1. "launch"; + // 2. "attach" with "processId"; + // 3. "attach" with "listen"; + // 4. "attach" with "connect" (or legacy "host"/"port"); + // + // For the first three, we want to spawn the debug adapter directly. + // For the last one, the adapter is already listening on the specified socket. + // When "debugServer" is used, the standard adapter factory takes care of it - no need to check here. + + if (configuration.request === 'attach') { + if (configuration.connect !== undefined) { + return new DebugAdapterServer( + configuration.connect.port, + configuration.connect.host ?? '127.0.0.1' + ); + } else if (configuration.port !== undefined) { + return new DebugAdapterServer(configuration.port, configuration.host ?? '127.0.0.1'); + } else if (configuration.listen === undefined && configuration.processId === undefined) { + throw new Error('"request":"attach" requires either "connect", "listen", or "processId"'); } - } else { - const pythonPath = await this.getPythonPath(configuration, session.workspaceFolder); - // If logToFile is set in the debug config then pass --log-dir when launching the debug adapter. + } + + const pythonPath = await this.getPythonPath(configuration, session.workspaceFolder); + if (pythonPath.length !== 0) { + if (configuration.request === 'attach' && configuration.processId !== undefined) { + sendTelemetryEvent(EventName.DEBUGGER_ATTACH_TO_LOCAL_PROCESS); + } + + // "logToFile" is not handled directly by the adapter - instead, we need to pass + // the corresponding CLI switch when spawning it. const logArgs = configuration.logToFile ? ['--log-dir', EXTENSION_ROOT_DIR] : []; + + if (configuration.debugAdapterPath !== undefined) { + return new DebugAdapterExecutable(pythonPath, [configuration.debugAdapterPath, ...logArgs]); + } + const debuggerPathToUse = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'lib', 'python', 'debugpy'); - if (pythonPath.length !== 0) { - if (processId) { - sendTelemetryEvent(EventName.DEBUGGER_ATTACH_TO_LOCAL_PROCESS); - } - - if (configuration.debugAdapterPath) { - return new DebugAdapterExecutable(pythonPath, [configuration.debugAdapterPath, ...logArgs]); - } - - if (await this.useNewDebugger(pythonPath)) { - sendTelemetryEvent(EventName.DEBUG_ADAPTER_USING_WHEELS_PATH, undefined, { usingWheels: true }); - return new DebugAdapterExecutable(pythonPath, [ - path.join(debuggerPathToUse, 'wheels', 'debugpy', 'adapter'), - ...logArgs - ]); - } else { - sendTelemetryEvent(EventName.DEBUG_ADAPTER_USING_WHEELS_PATH, undefined, { - usingWheels: false - }); - return new DebugAdapterExecutable(pythonPath, [ - path.join(debuggerPathToUse, 'no_wheels', 'debugpy', 'adapter'), - ...logArgs - ]); - } + if (await this.useNewDebugger(pythonPath)) { + sendTelemetryEvent(EventName.DEBUG_ADAPTER_USING_WHEELS_PATH, undefined, { usingWheels: true }); + return new DebugAdapterExecutable(pythonPath, [ + path.join(debuggerPathToUse, 'wheels', 'debugpy', 'adapter'), + ...logArgs + ]); + } else { + sendTelemetryEvent(EventName.DEBUG_ADAPTER_USING_WHEELS_PATH, undefined, { + usingWheels: false + }); + return new DebugAdapterExecutable(pythonPath, [ + path.join(debuggerPathToUse, 'no_wheels', 'debugpy', 'adapter'), + ...logArgs + ]); } } } else { diff --git a/src/test/debugger/extension/adapter/factory.unit.test.ts b/src/test/debugger/extension/adapter/factory.unit.test.ts index d8723b3ee244..903a072c0cba 100644 --- a/src/test/debugger/extension/adapter/factory.unit.test.ts +++ b/src/test/debugger/extension/adapter/factory.unit.test.ts @@ -206,37 +206,58 @@ suite('Debugging - Adapter Factory', () => { assert.deepEqual(descriptor, nodeExecutable); }); - test('Return Debug Adapter server if in DA experiment, configuration is attach and port is specified', async () => { + test('Return Debug Adapter server if in DA experiment, request is "attach", and port is specified directly', async () => { const session = createSession({ request: 'attach', port: 5678, host: 'localhost' }); const debugServer = new DebugAdapterServer(session.configuration.port, session.configuration.host); when(spiedInstance.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); - // Interpreter not needed for attach + // Interpreter not needed for host/port verify(interpreterService.getInterpreters(anything())).never(); assert.deepEqual(descriptor, debugServer); }); - test('Throw error if in DA experiment, configuration is attach, port is 0 and process ID is not specified', async () => { - const session = createSession({ request: 'attach', port: 0, host: 'localhost' }); + test('Return Debug Adapter server if in DA experiment, request is "attach", and connect is specified', async () => { + const session = createSession({ request: 'attach', connect: { port: 5678, host: 'localhost' } }); + const debugServer = new DebugAdapterServer( + session.configuration.connect.port, + session.configuration.connect.host + ); when(spiedInstance.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); - const promise = factory.createDebugAdapterDescriptor(session, nodeExecutable); + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); - await expect(promise).to.eventually.be.rejectedWith( - 'Port or processId must be specified for request type attach' - ); + // Interpreter not needed for connect + verify(interpreterService.getInterpreters(anything())).never(); + assert.deepEqual(descriptor, debugServer); + }); + + test('Return Debug Adapter executable if in DA experiment, request is "attach", and listen is specified', async () => { + const session = createSession({ request: 'attach', listen: { port: 5678, host: 'localhost' } }); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [ptvsdAdapterPathWithWheels]); + + when(spiedInstance.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + when(interpreterService.getActiveInterpreter(anything())).thenResolve(interpreter); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + assert.deepEqual(descriptor, debugExecutable); }); - test('Throw error if in DA experiment, configuration is attach and port and process ID are not specified', async () => { - const session = createSession({ request: 'attach', port: undefined, processId: undefined }); + test('Throw error if in DA experiment, request is "attach", and neither port, processId, listen, nor connect is specified', async () => { + const session = createSession({ + request: 'attach', + port: undefined, + processId: undefined, + listen: undefined, + connect: undefined + }); when(spiedInstance.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); const promise = factory.createDebugAdapterDescriptor(session, nodeExecutable); await expect(promise).to.eventually.be.rejectedWith( - 'Port or processId must be specified for request type attach' + '"request":"attach" requires either "connect", "listen", or "processId"' ); }); From 1fd5509195ad91e79133be2eead42245f1b93579 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Wed, 18 Mar 2020 21:25:13 -0700 Subject: [PATCH 5/5] Update version and change log (#10662) * Update version and change log * Fix flakey file system tests (#10541) Co-authored-by: Rich Chiodo --- CHANGELOG.md | 6 ++-- news/2 Fixes/10597.md | 1 - package-lock.json | 2 +- package.json | 2 +- .../platform/filesystem.functional.test.ts | 30 +++++++++++-------- 5 files changed, 23 insertions(+), 18 deletions(-) delete mode 100644 news/2 Fixes/10597.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e94bbda3067..75173afd8f19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 2020.3.0-rc (9 March 2020) +## 2020.3.0 (19 March 2020) ### Enhancements @@ -114,6 +114,8 @@ ([#10311](https://github.com/Microsoft/vscode-python/issues/10311)) 1. When you install missing dependencies for Jupyter successfully in an active interpreter also set that interpreter as the Jupyter selected interpreter. ([#10359](https://github.com/Microsoft/vscode-python/issues/10359)) +1. Ensure default `host` is not set, if `connect` or `listen` settings are available. + ([#10597](https://github.com/Microsoft/vscode-python/issues/10597)) ### Code Health @@ -135,8 +137,6 @@ ([#10182](https://github.com/Microsoft/vscode-python/issues/10182)) 1. Use debugpy in the core extension instead of ptvsd. ([#10184](https://github.com/Microsoft/vscode-python/issues/10184)) -1. Remove UI Tests. - ([#10192](https://github.com/Microsoft/vscode-python/issues/10192)) 1. Add telemetry for imports in notebooks. ([#10209](https://github.com/Microsoft/vscode-python/issues/10209)) 1. Update data science component to use `debugpy`. diff --git a/news/2 Fixes/10597.md b/news/2 Fixes/10597.md deleted file mode 100644 index c666d8d8d5b4..000000000000 --- a/news/2 Fixes/10597.md +++ /dev/null @@ -1 +0,0 @@ -Ensure default `host` is not set, if `connect` or `listen` settings are available. diff --git a/package-lock.json b/package-lock.json index 5a8f0b02651e..a72f68510a9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "python", - "version": "2020.3.0-rc", + "version": "2020.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 74591c67f73f..ed47fd15adbf 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "python", "displayName": "Python", "description": "Linting, Debugging (multi-threaded, remote), Intellisense, Jupyter Notebooks, code formatting, refactoring, unit tests, snippets, and more.", - "version": "2020.3.0-rc", + "version": "2020.3.0", "languageServerVersion": "0.5.30", "publisher": "ms-python", "author": { diff --git a/src/test/common/platform/filesystem.functional.test.ts b/src/test/common/platform/filesystem.functional.test.ts index f4de46ef3370..1d2a9a8f95ec 100644 --- a/src/test/common/platform/filesystem.functional.test.ts +++ b/src/test/common/platform/filesystem.functional.test.ts @@ -8,7 +8,8 @@ import * as fs from 'fs-extra'; import { convertStat, FileSystem, FileSystemUtils, RawFileSystem } from '../../../client/common/platform/fileSystem'; import { FileSystemPaths, FileSystemPathUtils } from '../../../client/common/platform/fs-paths'; import { FileType } from '../../../client/common/platform/types'; -import { sleep } from '../../../client/common/utils/async'; +import { createDeferred, sleep } from '../../../client/common/utils/async'; +import { noop } from '../../../client/common/utils/misc'; import { assertDoesNotExist, assertFileText, @@ -267,13 +268,24 @@ suite('FileSystem - raw', () => { } }); + async function writeToStream(filename: string, write: (str: fs.WriteStream) => void) { + const closeDeferred = createDeferred(); + const stream = fileSystem.createWriteStream(filename); + stream.on('close', () => closeDeferred.resolve()); + write(stream); + stream.end(); + stream.close(); + stream.destroy(); + await closeDeferred.promise; + return stream; + } + test('returns the correct WriteStream', async () => { const filename = await fix.resolve('x/y/z/spam.py'); const expected = fs.createWriteStream(filename); expected.destroy(); - const stream = fileSystem.createWriteStream(filename); - stream.destroy(); + const stream = await writeToStream(filename, noop); expect(stream.path).to.deep.equal(expected.path); }); @@ -283,9 +295,7 @@ suite('FileSystem - raw', () => { await assertDoesNotExist(filename); const data = 'line1\nline2\n'; - const stream = fileSystem.createWriteStream(filename); - stream.write(data); - stream.destroy(); + await writeToStream(filename, s => s.write(data)); await assertFileText(filename, data); }); @@ -294,9 +304,7 @@ suite('FileSystem - raw', () => { const filename = await fix.resolve('x/y/z/spam.py'); const data = '... 😁 ...'; - const stream = fileSystem.createWriteStream(filename); - stream.write(data); - stream.destroy(); + await writeToStream(filename, s => s.write(data)); await assertFileText(filename, data); }); @@ -305,9 +313,7 @@ suite('FileSystem - raw', () => { const filename = await fix.createFile('x/y/z/spam.py', '...'); const data = 'line1\nline2\n'; - const stream = fileSystem.createWriteStream(filename); - stream.write(data); - stream.destroy(); + await writeToStream(filename, s => s.write(data)); await assertFileText(filename, data); });