Skip to content

Commit 8633a8d

Browse files
rchiodorchiodo
authored
Implement IJupyterVariables for the debugger (microsoft#11543)
* Preliminary getting variables from the debugger * Implement updating as stepping * Get debugger and regular kernel to use same scripts * Put back old code and create experiment to enable/disable this * Implement experiment and begin on test for debugger * Finish fixing tests * Add news entry * Update comment * Code review feedback * Fix crash during functional test on new setting * Fix the ipython unit tests * Fix formatting
1 parent 4b3111b commit 8633a8d

41 files changed

Lines changed: 2095 additions & 1322 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.sonarcloud.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
sonar.sources=src/client,src/datascience-ui
22
sonar.tests=src/test
33
sonar.cfamily.build-wrapper-output.bypass=true
4-
sonar.cpd.exclusions=src/datascience-ui/**/redux/actions.ts,src/client/**/raw-kernel/rawKernel.ts
4+
sonar.cpd.exclusions=src/datascience-ui/**/redux/actions.ts,src/client/**/raw-kernel/rawKernel.ts,src/client/datascience/jupyter/*ariable*.ts

‎.vscode/launch.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
// Enable this to try out new experiments locally
2727
"XVSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE" : "1",
2828
// Enable this to log telemetry to the output during debugging
29-
"XVSC_PYTHON_LOG_TELEMETRY": "1"
29+
"XVSC_PYTHON_LOG_TELEMETRY": "1",
30+
// Enable this to log debugger output. Directory must exist ahead of time
31+
"XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output_Ex"
3032
}
3133
},
3234
{
@@ -275,7 +277,9 @@
275277
// Remove 'X' to turn on all logging in the debug output
276278
"XVSC_PYTHON_FORCE_LOGGING": "1",
277279
// Remove `X` prefix and update path to test with real python interpreter (for DS functional tests).
278-
"XCI_PYTHON_PATH": "<Python Path>"
280+
"XCI_PYTHON_PATH": "<Python Path>",
281+
// Remove 'X' prefix to dump output for debugger. Directory has to exist prior to launch
282+
"XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output"
279283
},
280284
"outFiles": [
281285
"${workspaceFolder}/out/**/*.js"

‎experiments.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,5 +172,18 @@
172172
"salt": "DeprecatePythonPath",
173173
"min": 100,
174174
"max": 100
175+
},
176+
{
177+
"name": "RunByLine - control",
178+
"salt": "RunByLine",
179+
"min": 0,
180+
"max": 100
181+
},
182+
{
183+
"name": "RunByLine - experiment",
184+
"salt": "RunByLine",
185+
"max": 0,
186+
"min": 0
175187
}
188+
176189
]

‎news/3 Code Health/11542.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implement an IJupyterVariables provider for the debugger.

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1604,6 +1604,7 @@
16041604
"CollectLSRequestTiming - experiment",
16051605
"CollectNodeLSRequestTiming - experiment",
16061606
"DeprecatePythonPath - experiment",
1607+
"RunByLine - experiment",
16071608
"All"
16081609
]
16091610
},
@@ -1628,6 +1629,7 @@
16281629
"CollectLSRequestTiming - experiment",
16291630
"CollectNodeLSRequestTiming - experiment",
16301631
"DeprecatePythonPath - experiment",
1632+
"RunByLine - experiment",
16311633
"All"
16321634
]
16331635
},

‎pythonFiles/tests/ipython/scripts.py‎

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ def execute_script(file, replace_dict=dict([])):
4343
return result.success
4444

4545

46+
def execute_code(code):
47+
# Execute this script as a cell
48+
result = get_ipython().run_cell(code)
49+
return result
50+
51+
4652
def get_variables(capsys):
4753
path = os.path.dirname(os.path.abspath(__file__))
4854
file = os.path.abspath(os.path.join(path, "./getJupyterVariableList.py"))
@@ -74,35 +80,41 @@ def get_variable_value(variables, name, capsys):
7480
def get_data_frame_info(variables, name, capsys):
7581
varJson = find_variable_json(variables, name)
7682
path = os.path.dirname(os.path.abspath(__file__))
77-
file = os.path.abspath(
78-
os.path.join(
79-
path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py"
80-
)
83+
syspath = os.path.abspath(
84+
os.path.join(path, "../../vscode_datascience_helpers/dataframes")
8185
)
82-
keys = dict([("_VSCode_JupyterTestValue", json.dumps(varJson))])
83-
if execute_script(file, keys):
86+
syscode = 'import sys\nsys.path.append("{0}")'.format(syspath.replace("\\", "\\\\"))
87+
importcode = "import vscodeGetDataFrameInfo\nprint(vscodeGetDataFrameInfo._VSCODE_getDataFrameInfo({0}))".format(
88+
name
89+
)
90+
result = execute_code(syscode)
91+
if not result.success:
92+
result.raise_error()
93+
result = execute_code(importcode)
94+
if result.success:
8495
read_out = capsys.readouterr()
85-
return json.loads(read_out.out)
96+
info = json.loads(read_out.out[0:-1])
97+
varJson.update(info)
98+
return varJson
8699
else:
87-
raise Exception("Get dataframe info failed.")
100+
result.raise_error()
88101

89102

90103
def get_data_frame_rows(varJson, start, end, capsys):
91104
path = os.path.dirname(os.path.abspath(__file__))
92-
file = os.path.abspath(
93-
os.path.join(
94-
path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameRows.py"
95-
)
105+
syspath = os.path.abspath(
106+
os.path.join(path, "../../vscode_datascience_helpers/dataframes")
96107
)
97-
keys = dict(
98-
[
99-
("_VSCode_JupyterTestValue", json.dumps(varJson)),
100-
("_VSCode_JupyterStartRow", str(start)),
101-
("_VSCode_JupyterEndRow", str(end)),
102-
]
108+
syscode = 'import sys\nsys.path.append("{0}")'.format(syspath.replace("\\", "\\\\"))
109+
importcode = "import vscodeGetDataFrameRows\nprint(vscodeGetDataFrameRows._VSCODE_getDataFrameRows({0}, {1}, {2}))".format(
110+
varJson["name"], start, end
103111
)
104-
if execute_script(file, keys):
112+
result = execute_code(syscode)
113+
if not result.success:
114+
result.raise_error()
115+
result = execute_code(importcode)
116+
if result.success:
105117
read_out = capsys.readouterr()
106-
return json.loads(read_out.out)
118+
return json.loads(read_out.out[0:-1])
107119
else:
108-
raise Exception("Getting dataframe rows failed.")
120+
result.raise_error()

‎pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import pandas as _VSCODE_pd
2+
import builtins as _VSCODE_builtins
3+
4+
# Function that converts the var passed in into a pandas data frame if possible
5+
def _VSCODE_convertToDataFrame(df):
6+
if isinstance(df, list):
7+
df = _VSCODE_pd.DataFrame(df)
8+
elif isinstance(df, _VSCODE_pd.Series):
9+
df = _VSCODE_pd.Series.to_frame(df)
10+
elif isinstance(df, dict):
11+
df = _VSCODE_pd.Series(df)
12+
df = _VSCODE_pd.Series.to_frame(df)
13+
elif hasattr(df, "toPandas"):
14+
df = df.toPandas()
15+
else:
16+
try:
17+
temp = _VSCODE_pd.DataFrame(df)
18+
df = temp
19+
except:
20+
pass
21+
return df
22+
23+
24+
# Function to compute row count for a value
25+
def _VSCODE_getRowCount(var):
26+
if hasattr(var, "shape"):
27+
try:
28+
# Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
29+
if isinstance(var.shape, tuple):
30+
return var.shape[0]
31+
except TypeError:
32+
return 0
33+
elif hasattr(var, "__len__"):
34+
try:
35+
return _VSCODE_builtins.len(var)
36+
except TypeError:
37+
return 0

‎pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Query Jupyter server for the info about a dataframe
2+
import json as _VSCODE_json
3+
import pandas as _VSCODE_pd
4+
import pandas.io.json as _VSCODE_pd_json
5+
import builtins as _VSCODE_builtins
6+
import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
7+
8+
# Function to do our work. It will return the object
9+
def _VSCODE_getDataFrameInfo(df):
10+
df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df)
11+
rowCount = _VSCODE_dataFrameHelpers._VSCODE_getRowCount(df)
12+
13+
# If any rows, use pandas json to convert a single row to json. Extract
14+
# the column names and types from the json so we match what we'll fetch when
15+
# we ask for all of the rows
16+
if rowCount:
17+
try:
18+
row = df.iloc[0:1]
19+
json_row = _VSCODE_pd_json.to_json(None, row, date_format="iso")
20+
columnNames = list(_VSCODE_json.loads(json_row))
21+
except:
22+
columnNames = list(df)
23+
else:
24+
columnNames = list(df)
25+
26+
# Compute the index column. It may have been renamed
27+
indexColumn = df.index.name if df.index.name else "index"
28+
columnTypes = _VSCODE_builtins.list(df.dtypes)
29+
30+
# Make sure the index column exists
31+
if indexColumn not in columnNames:
32+
columnNames.insert(0, indexColumn)
33+
columnTypes.insert(0, "int64")
34+
35+
# Then loop and generate our output json
36+
columns = []
37+
for n in _VSCODE_builtins.range(0, _VSCODE_builtins.len(columnNames)):
38+
column_type = columnTypes[n]
39+
column_name = str(columnNames[n])
40+
colobj = {}
41+
colobj["key"] = column_name
42+
colobj["name"] = column_name
43+
colobj["type"] = str(column_type)
44+
columns.append(colobj)
45+
46+
# Save this in our target
47+
target = {}
48+
target["columns"] = columns
49+
target["indexColumn"] = indexColumn
50+
target["rowCount"] = rowCount
51+
52+
# return our json object as a string
53+
return _VSCODE_json.dumps(target)

‎pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Query for the rows of a data frame
2+
import pandas.io.json as _VSCODE_pd_json
3+
import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
4+
5+
# Function to retrieve a set of rows for a data frame
6+
def _VSCODE_getDataFrameRows(df, start, end):
7+
df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df)
8+
9+
# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON
10+
rows = df.iloc[start:end]
11+
return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso")

‎src/client/common/experimentGroups.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,12 @@ export enum LocalZMQKernel {
4848
experiment = 'LocalZMQKernel - experiment'
4949
}
5050

51+
// Experiment for supporting run by line in data science notebooks
52+
export enum RunByLine {
53+
control = 'RunByLine - control',
54+
experiment = 'RunByLine - experiment'
55+
}
56+
5157
/**
5258
* Experiment to check whether to to use a terminal to generate the environment variables of activated environments.
5359
*

0 commit comments

Comments
 (0)