Skip to content

Commit 1338fc3

Browse files
rchiodorchiodo
authored
Add support for dataframes into the DataExplorer (microsoft#4969)
For #4677 <!-- If an item below does not apply to you, then go ahead and check it off as "done" and strikethrough the text, e.g.: - [x] ~Has unit tests & system/integration tests~ --> - [x] Pull request represents a single change (i.e. not fixing disparate/unrelated things in a single PR) - [x] Title summarizes what is changing - [x] Has a [news entry](https://github.com/Microsoft/vscode-python/tree/master/news) file (remember to thank yourself!) - [ ] Has sufficient logging. - [ ] Has telemetry for enhancements. - [ ] Unit tests & system/integration tests are added/updated - [ ] [Test plan](https://github.com/Microsoft/vscode-python/blob/master/.github/test_plan.md) is updated as appropriate - [ ] [`package-lock.json`](https://github.com/Microsoft/vscode-python/blob/master/package-lock.json) has been regenerated by running `npm install` (if dependencies have changed) - [ ] The wiki is updated with any design decisions/details.
1 parent 419ecb1 commit 1338fc3

33 files changed

Lines changed: 1766 additions & 1032 deletions

‎news/1 Enhancements/4677.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add preliminary support for viewing dataframes.

‎package-lock.json‎

Lines changed: 70 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2283,6 +2283,7 @@
22832283
"rxjs": "^5.5.9",
22842284
"semver": "^5.5.0",
22852285
"stack-trace": "0.0.10",
2286+
"strip-ansi": "^5.2.0",
22862287
"strip-json-comments": "^2.0.1",
22872288
"sudo-prompt": "^8.2.0",
22882289
"tmp": "^0.0.29",

‎package.nls.json‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,5 +222,11 @@
222222
"LanguageService.extractionFailedOutputMessage": "extraction failed",
223223
"LanguageService.extractionCompletedOutputMessage": "complete",
224224
"LanguageService.extractionDoneOutputMessage": "done",
225-
"LanguageService.reloadVSCodeIfSeachPathHasChanged": "Search paths have changed for this Python interpreter. Please reload the extension to ensure that the IntelliSense works correctly"
225+
"LanguageService.reloadVSCodeIfSeachPathHasChanged": "Search paths have changed for this Python interpreter. Please reload the extension to ensure that the IntelliSense works correctly",
226+
"DataScience.dataExplorerInvalidVariableFormat" : "'{0}' is not an active variable.",
227+
"DataScience.jupyterGetVariablesExecutionError" : "Failure during variable extraction:\r\n{0}",
228+
"DataScience.loadingMessage" : "loading ...",
229+
"DataScience.noRowsInDataExplorer" : "Fetching data ...",
230+
"DataScience.pandasTooOldForViewingFormat" : "Python package 'pandas' is version {0}. Version 0.20 or greater is required for viewing data.",
231+
"DataScience.pandasRequiredForViewing" : "Python package 'pandas' is required for viewing data."
226232
}

‎pythonFiles/datascience/getJupyterVariableDataFrameInfo.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Query Jupyter server for the info about a dataframe
2+
import json as _VSCODE_json
3+
4+
# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable
5+
# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
6+
_VSCODE_targetVariable = _VSCODE_json.loads('_VSCode_JupyterTestValue')
7+
_VSCODE_evalResult = eval(_VSCODE_targetVariable['name'])
8+
9+
# First list out the columns of the data frame (assuming it is one for now)
10+
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
11+
_VSCODE_columnNames = list(_VSCODE_evalResult)
12+
13+
# Make sure we have an index column (see code in getJupyterVariableDataFrameRows.py)
14+
if 'index' not in _VSCODE_columnNames:
15+
_VSCODE_columnNames.insert(0, 'index')
16+
_VSCODE_columnTypes.insert(0, 'int64')
17+
18+
# Then loop and generate our output json
19+
_VSCODE_columns = []
20+
for n in range(0, len(_VSCODE_columnNames)):
21+
c = _VSCODE_columnNames[n]
22+
t = _VSCODE_columnTypes[n]
23+
_VSCODE_colobj = {}
24+
_VSCODE_colobj['key'] = c
25+
_VSCODE_colobj['name'] = c
26+
_VSCODE_colobj['type'] = str(t)
27+
_VSCODE_columns.append(_VSCODE_colobj)
28+
29+
del _VSCODE_columnNames
30+
del _VSCODE_columnTypes
31+
32+
# Save this in our target
33+
_VSCODE_targetVariable['columns'] = _VSCODE_columns
34+
del _VSCODE_columns
35+
36+
# Figure out shape if not already there
37+
if 'shape' not in _VSCODE_targetVariable:
38+
_VSCODE_targetVariable['shape'] = str(_VSCODE_evalResult.shape)
39+
40+
# Row count is actually embedded in shape. Should be the second number
41+
import re as _VSCODE_re
42+
_VSCODE_regex = r"\(\s*(\d+),\s*(\d+)\s*\)"
43+
_VSCODE_matches = _VSCODE_re.search(_VSCODE_regex, _VSCODE_targetVariable['shape'])
44+
if (_VSCODE_matches):
45+
_VSCODE_targetVariable['rowCount'] = int(_VSCODE_matches[1])
46+
del _VSCODE_matches
47+
else:
48+
_VSCODE_targetVariable['rowCount'] = 0
49+
del _VSCODE_regex
50+
51+
# Transform this back into a string
52+
print(_VSCODE_json.dumps(_VSCODE_targetVariable))

‎pythonFiles/datascience/getJupyterVariableDataFrameRows.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Query Jupyter server for the rows of a data frame
2+
import json as _VSCODE_json
3+
import pandas.io.json as _VSCODE_pd_json
4+
5+
# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable
6+
# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
7+
_VSCODE_targetVariable = _VSCODE_json.loads('_VSCode_JupyterTestValue')
8+
_VSCODE_evalResult = eval(_VSCODE_targetVariable['name'])
9+
10+
# _VSCode_JupyterStartRow and _VSCode_JupyterEndRow should be replaced dynamically with the literals
11+
# for our start and end rows
12+
_VSCODE_startRow = max(_VSCode_JupyterStartRow, 0)
13+
_VSCODE_endRow = min(_VSCode_JupyterEndRow, _VSCODE_targetVariable['rowCount'])
14+
15+
# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON
16+
_VSCODE_rows = df.iloc[_VSCODE_startRow:_VSCODE_endRow]
17+
_VSCODE_result = _VSCODE_pd_json.to_json(None, _VSCODE_rows, orient='table', date_format='iso')
18+
print(_VSCODE_result)
19+
20+
# Cleanup our variables
21+
del _VSCODE_endRow
22+
del _VSCODE_startRow
23+
del _VSCODE_rows
24+
del _VSCODE_result

‎pythonFiles/datascience/getJupyterVariableList.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# Query Jupyter server for defined variables list
22
# Tested on 2.7 and 3.6
33
from sys import getsizeof
4-
import json
4+
import json as _VSCODE_json
55

66
# who_ls is a Jupyter line magic to fetch currently defined vars
77
_VSCode_JupyterVars = %who_ls
88

9-
print(json.dumps([{'name': var,
10-
'type': type(eval(var)).__name__,
11-
'size': getsizeof(var),
12-
'expensive': True
13-
} for var in _VSCode_JupyterVars]))
9+
print(_VSCODE_json.dumps([{'name': var,
10+
'type': type(eval(var)).__name__,
11+
'size': getsizeof(var),
12+
'expensive': True
13+
} for var in _VSCode_JupyterVars]))

‎pythonFiles/datascience/getJupyterVariableValue.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Query Jupyter server for the value of a variable
2-
import json
2+
import json as _VSCODE_json
33
_VSCODE_max_len = 200
44
# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable
55
# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
6-
_VSCODE_targetVariable = json.loads('_VSCode_JupyterTestValue')
6+
_VSCODE_targetVariable = _VSCODE_json.loads('_VSCode_JupyterTestValue')
77

88
_VSCODE_evalResult = eval(_VSCODE_targetVariable['name'])
99

@@ -22,4 +22,4 @@
2222
else:
2323
_VSCODE_targetVariable['value'] = _VSCODE_targetValue
2424

25-
print(json.dumps(_VSCODE_targetVariable))
25+
print(_VSCODE_json.dumps(_VSCODE_targetVariable))

‎src/client/common/utils/localize.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,13 @@ export namespace DataScience {
143143
export const liveShareServiceFailure = localize('DataScience.liveShareServiceFailure', 'Failure starting \'{0}\' service during live share connection.');
144144
export const documentMismatch = localize('DataScience.documentMismatch', 'Cannot run cells, duplicate documents for {0} found.');
145145
export const jupyterGetVariablesBadResults = localize('DataScience.jupyterGetVariablesBadResults', 'Failed to fetch variable info from the Jupyter server.');
146+
export const dataExplorerInvalidVariableFormat = localize('DataScience.dataExplorerInvalidVariableFormat', '\'{0}\' is not an active variable.');
146147
export const pythonInteractiveCreateFailed = localize('DataScience.pythonInteractiveCreateFailed', 'Failure to create a \'Python Interactive\' window. Try reinstalling the Python extension.');
147-
148+
export const jupyterGetVariablesExecutionError = localize('DataScience.jupyterGetVariablesExecutionError', 'Failure during variable extraction: \r\n{0}');
149+
export const loadingMessage = localize('DataScience.loadingMessage', 'loading ...');
150+
export const noRowsInDataExplorer = localize('DataScience.noRowsInDataExplorer', 'Fetching data ...');
151+
export const pandasTooOldForViewingFormat = localize('DataScience.pandasTooOldForViewingFormat', 'Python package \'pandas\' is version {0}. Version 0.20 or greater is required for viewing data.');
152+
export const pandasRequiredForViewing = localize('DataScience.pandasRequiredForViewing', 'Python package \'pandas\' is required for viewing data.');
148153
}
149154

150155
export namespace DebugConfigurationPrompts {

‎src/client/datascience/codeCssGenerator.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ export class CodeCssGenerator implements ICodeCssGenerator {
122122
const numericStyle = this.getScopeStyle(tokenColors, 'constant.numeric');
123123
const stringStyle = this.getScopeStyle(tokenColors, 'string');
124124
const keywordStyle = this.getScopeStyle(tokenColors, 'keyword.control', 'keyword');
125-
const operatorStyle = this.getScopeStyle(tokenColors, 'keyword.operator');
125+
const operatorStyle = this.getScopeStyle(tokenColors, 'keyword.operator', 'keyword');
126126
const variableStyle = this.getScopeStyle(tokenColors, 'variable');
127127
// const atomic = this.getScopeColor(tokenColors, 'atomic');
128128
const builtinStyle = this.getScopeStyle(tokenColors, 'support.function');

0 commit comments

Comments
 (0)