Skip to content

Commit ecf35d8

Browse files
ericsnowcurrentlyericsnowcurrently
authored
Functional tests for the pytest adapter. (#5161)
(for #4739)
1 parent 3f57bf0 commit ecf35d8

37 files changed

Lines changed: 2085 additions & 103 deletions

‎CONTRIBUTING.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,22 @@ const grep = '[The suite name of your *test.ts file]'; // IS_CI_SERVER &&...
129129

130130
And be sure to escape any grep-sensitive characters in your suite name (and to remove the change from src/test/index.ts before you submit).
131131

132+
### Testing Python Scripts
133+
134+
The extension has a number of scripts in ./pythonFiles. Tests for these
135+
scripts are found in ./pythonFiles/tests. To run those tests:
136+
137+
* `python2.7 pythonFiles/tests/run_all.py`
138+
* `python3 -m pythonFiles.tests`
139+
140+
By default, functional tests are included. To exclude them:
141+
142+
`python3 -m pythonFiles.tests --no-functional`
143+
144+
To run only the functional tests:
145+
146+
`python3 -m pythonFiles.tests --functional`
147+
132148
### Standard Debugging
133149

134150
Clone the repo into any directory, open that directory in VSCode, and use the `Launch Extension` launch option within VSCode.

‎news/3 Code Health/4739.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add functional tests for pytest adapter script.

‎pythonFiles/testing_tools/adapter/pytest.py‎

Lines changed: 80 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from __future__ import absolute_import
55

66
import os.path
7+
import sys
78

89
import pytest
910

@@ -141,6 +142,7 @@ def _ensure_parent(self, path, parentid, suiteids):
141142
rootdir, parent)
142143
self._parents[(rootdir, parentid)] = funcinfo
143144
elif parent != parentid:
145+
print(parent, parentid)
144146
# TODO: What to do?
145147
raise NotImplementedError
146148
return parentid
@@ -168,10 +170,12 @@ def _ensure_file(self, rootdir, relfile):
168170
def _ensure_suites(self, fullsuite, rootdir, fileid, suiteids):
169171
if not fullsuite:
170172
if suiteids:
173+
print(suiteids)
171174
# TODO: What to do?
172175
raise NotImplementedError
173176
return None
174177
if len(suiteids) != fullsuite.count('.') + 1:
178+
print(suiteids)
175179
# TODO: What to do?
176180
raise NotImplementedError
177181

@@ -210,13 +214,12 @@ def _parse_item(item, _normcase, _pathsep):
210214
#_debug_item(item, showsummary=True)
211215
kind, _ = _get_item_kind(item)
212216
# Figure out the func, suites, and subs.
213-
(fileid, suiteids, suites, funcid, basename, parameterized
214-
) = _parse_node_id(item.nodeid, kind)
217+
(nodeid, fileid, suiteids, suites, funcid, basename, parameterized
218+
) = _parse_node_id(item.nodeid, kind, _pathsep, _normcase)
215219
if kind == 'function':
216220
funcname = basename
217-
if funcid and item.function.__name__ != funcname:
218-
# TODO: What to do?
219-
raise NotImplementedError
221+
# Note: funcname does not necessarily match item.function.__name__.
222+
# This can result from importing a test function from another module.
220223
if suites:
221224
testfunc = '.'.join(suites) + '.' + funcname
222225
else:
@@ -226,32 +229,23 @@ def _parse_item(item, _normcase, _pathsep):
226229
funcname = None
227230

228231
# Figure out the file.
232+
relfile = _normcase(fileid)
229233
fspath = str(item.fspath)
230-
if not fspath.endswith(_pathsep + fileid):
234+
if not _normcase(fspath).endswith(relfile[1:]):
235+
print(fspath)
236+
print(relfile)
231237
raise NotImplementedError
232-
filename = fspath[-len(fileid):]
233-
testroot = str(item.fspath)[:-len(fileid)].rstrip(_pathsep)
234-
if _pathsep in filename:
235-
relfile = filename
236-
else:
237-
relfile = '.' + _pathsep + filename
238-
srcfile, lineno, fullname = item.location
239-
if srcfile != fileid:
240-
# pytest supports discovery of tests imported from other
241-
# modules. This is reflected by a different filename
242-
# in item.location.
243-
if _normcase(fileid) == _normcase(srcfile):
244-
srcfile = fileid
245-
else:
246-
srcfile = relfile
247-
location = '{}:{}'.format(srcfile, lineno)
238+
testroot = str(item.fspath)[:-len(relfile) + 1]
239+
location, fullname = _get_location(item, relfile, _normcase, _pathsep)
248240
if kind == 'function':
249241
if testfunc and fullname != testfunc + parameterized:
250-
print(fullname, testfunc)
242+
print(item.nodeid)
243+
print(fullname, suites, testfunc)
251244
# TODO: What to do?
252245
raise NotImplementedError
253246
elif kind == 'doctest':
254247
if testfunc and fullname != testfunc + parameterized:
248+
print(item.nodeid)
255249
print(fullname, testfunc)
256250
# TODO: What to do?
257251
raise NotImplementedError
@@ -280,7 +274,7 @@ def _parse_item(item, _normcase, _pathsep):
280274
# TODO: Support other markers?
281275

282276
test = TestInfo(
283-
id=item.nodeid,
277+
id=nodeid,
284278
name=item.name,
285279
path=TestPath(
286280
root=testroot,
@@ -295,11 +289,62 @@ def _parse_item(item, _normcase, _pathsep):
295289
return test, suiteids
296290

297291

298-
def _parse_node_id(nodeid, kind='function'):
292+
def _get_location(item, relfile, _normcase, _pathsep):
293+
srcfile, lineno, fullname = item.location
294+
srcfile = _normcase(srcfile)
295+
if srcfile in (relfile, relfile[len(_pathsep) + 1:]):
296+
srcfile = relfile
297+
else:
298+
# pytest supports discovery of tests imported from other
299+
# modules. This is reflected by a different filename
300+
# in item.location.
301+
srcfile, lineno = _find_location(
302+
srcfile, lineno, relfile, item.function, _pathsep)
303+
if not srcfile.startswith('.' + _pathsep):
304+
srcfile = '.' + _pathsep + srcfile
305+
# from pytest, line numbers are 0-based
306+
location = '{}:{}'.format(srcfile, int(lineno) + 1)
307+
return location, fullname
308+
309+
310+
def _find_location(srcfile, lineno, relfile, func, _pathsep):
311+
if sys.version_info > (3,):
312+
return srcfile, lineno
313+
if (_pathsep + 'unittest' + _pathsep + 'case.py') not in srcfile:
314+
return srcfile, lineno
315+
316+
# Unwrap the decorator (e.g. unittest.skip).
317+
srcfile = relfile
318+
lineno = -1
319+
try:
320+
func = func.__closure__[0].cell_contents
321+
except (IndexError, AttributeError):
322+
return srcfile, lineno
323+
else:
324+
if callable(func) and func.__code__.co_filename.endswith(relfile[1:]):
325+
lineno = func.__code__.co_firstlineno - 1
326+
return srcfile, lineno
327+
328+
329+
def _parse_node_id(nodeid, kind, _pathsep, _normcase):
330+
if not nodeid.startswith('.' + _pathsep):
331+
nodeid = '.' + _pathsep + nodeid
332+
while '::()::' in nodeid:
333+
nodeid = nodeid.replace('::()::', '::')
334+
335+
fileid, _, remainder = nodeid.partition('::')
336+
if not fileid or not remainder:
337+
print(nodeid)
338+
# TODO: Unexpected! What to do?
339+
raise NotImplementedError
340+
fileid = _normcase(fileid)
341+
nodeid = fileid + '::' + remainder
342+
299343
if kind == 'doctest':
300344
try:
301345
parentid, name = nodeid.split('::')
302346
except ValueError:
347+
print(nodeid)
303348
# TODO: Unexpected! What to do?
304349
raise NotImplementedError
305350
funcid = None
@@ -309,26 +354,30 @@ def _parse_node_id(nodeid, kind='function'):
309354
if nodeid.endswith(']'):
310355
funcid, sep, parameterized = nodeid.partition('[')
311356
if not sep:
357+
print(nodeid)
312358
# TODO: Unexpected! What to do?
313359
raise NotImplementedError
314360
parameterized = sep + parameterized
315361
else:
316362
funcid = nodeid
317-
318363
parentid, _, name = funcid.rpartition('::')
319-
if not name:
364+
if not parentid or not name:
365+
print(parentid, name)
320366
# TODO: What to do? We expect at least a filename and a function
321367
raise NotImplementedError
322368

323369
suites = []
324370
suiteids = []
325371
while '::' in parentid:
326-
suiteids.insert(0, parentid)
327-
parentid, _, suitename = parentid.rpartition('::')
372+
fullid = parentid
373+
parentid, _, suitename = fullid.rpartition('::')
374+
suiteids.insert(0, fullid)
328375
suites.insert(0, suitename)
329-
fileid = parentid
376+
if parentid != fileid:
377+
print(nodeid)
378+
print(parentid, fileid)
330379

331-
return fileid, suiteids, suites, funcid, name, parameterized
380+
return nodeid, fileid, suiteids, suites, funcid, name, parameterized
332381

333382

334383
def _get_item_kind(item):

‎pythonFiles/tests/__main__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@
1010

1111
TEST_ROOT = os.path.dirname(__file__)
1212
SRC_ROOT = os.path.dirname(TEST_ROOT)
13+
PROJECT_ROOT = os.path.dirname(SRC_ROOT)
1314
DATASCIENCE_ROOT = os.path.join(SRC_ROOT, 'datascience')
1415
TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, 'testing_tools')
1516

1617

1718
def parse_args():
1819
parser = argparse.ArgumentParser()
1920
# To mark a test as functional: (decorator) @pytest.mark.functional
21+
parser.add_argument('--functional', dest='markers',
22+
action='append_const', const='functional')
2023
parser.add_argument('--no-functional', dest='markers',
2124
action='append_const', const='not functional')
2225
args, remainder = parser.parse_known_args()

‎pythonFiles/tests/testing_tools/adapter/.data/complex/README.md‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
2+
## Directory Structure
3+
4+
```
5+
pythonFiles/tests/testing_tools/adapter/.data/
6+
tests/ # test root
7+
test_doctest.txt
8+
test_pytest.py
9+
test_unittest.py
10+
test_mixed.py
11+
spam.py # note: no "test_" prefix, but contains tests
12+
test_foo.py
13+
test_42.py
14+
test_42-43.py # note the hyphen
15+
testspam.py
16+
v/
17+
__init__.py
18+
spam.py
19+
test_eggs.py
20+
test_ham.py
21+
test_spam.py
22+
w/
23+
# no __init__.py
24+
test_spam.py
25+
test_spam_ex.py
26+
x/y/z/ # each with a __init__.py
27+
test_ham.py
28+
a/
29+
__init__.py
30+
test_spam.py
31+
b/
32+
__init__.py
33+
test_spam.py
34+
```
35+
36+
## Tests (and Suites)
37+
38+
basic:
39+
40+
* `./test_foo.py::test_simple`
41+
* `./test_pytest.py::test_simple`
42+
* `./test_pytest.py::TestSpam::test_simple`
43+
* `./test_pytest.py::TestSpam::TestHam::TestEggs::test_simple`
44+
* `./test_pytest.py::TestEggs::test_simple`
45+
* `./test_pytest.py::TestParam::test_simple`
46+
* `./test_mixed.py::test_top_level`
47+
* `./test_mixed.py::MyTests::test_simple`
48+
* `./test_mixed.py::TestMySuite::test_simple`
49+
* `./test_unittest.py::MyTests::test_simple`
50+
* `./test_unittest.py::OtherTests::test_simple`
51+
* `./x/y/z/test_ham.py::test_simple`
52+
* `./x/y/z/a/test_spam.py::test_simple`
53+
* `./x/y/z/b/test_spam.py::test_simple`
54+
55+
failures:
56+
57+
* `./test_pytest.py::test_failure`
58+
* `./test_pytest.py::test_runtime_failed`
59+
* `./test_pytest.py::test_raises`
60+
61+
skipped:
62+
63+
* `./test_mixed.py::test_skipped`
64+
* `./test_mixed.py::MyTests::test_skipped`
65+
* `./test_pytest.py::test_runtime_skipped`
66+
* `./test_pytest.py::test_skipped`
67+
* `./test_pytest.py::test_maybe_skipped`
68+
* `./test_pytest.py::SpamTests::test_skipped`
69+
* `./test_pytest.py::test_param_13_markers[???]`
70+
* `./test_pytest.py::test_param_13_skipped[*]`
71+
* `./test_unittest.py::MyTests::test_skipped`
72+
* (`./test_unittest.py::MyTests::test_maybe_skipped`)
73+
* (`./test_unittest.py::MyTests::test_maybe_not_skipped`)
74+
75+
in namespace package:
76+
77+
* `./w/test_spam.py::test_simple`
78+
* `./w/test_spam_ex.py::test_simple`
79+
80+
filename oddities:
81+
82+
* `./test_42.py::test_simple`
83+
* `./test_42-43.py::test_simple`
84+
* (`./testspam.py::test_simple` not discovered by default)
85+
* (`./spam.py::test_simple` not discovered)
86+
87+
imports discovered:
88+
89+
* `./v/test_eggs.py::test_simple`
90+
* `./v/test_eggs.py::TestSimple::test_simple`
91+
* `./v/test_ham.py::test_simple`
92+
* `./v/test_ham.py::test_not_hard`
93+
* `./v/test_spam.py::test_simple`
94+
* `./v/test_spam.py::test_simpler`
95+
96+
subtests:
97+
98+
* `./test_pytest.py::test_dynamic_*`
99+
* `./test_pytest.py::test_param_01[]`
100+
* `./test_pytest.py::test_param_11[1]`
101+
* `./test_pytest.py::test_param_13[*]`
102+
* `./test_pytest.py::test_param_13_markers[*]`
103+
* `./test_pytest.py::test_param_13_repeat[*]`
104+
* `./test_pytest.py::test_param_13_skipped[*]`
105+
* `./test_pytest.py::test_param_23_13[*]`
106+
* `./test_pytest.py::test_param_23_raises[*]`
107+
* `./test_pytest.py::test_param_33[*]`
108+
* `./test_pytest.py::test_param_33_ids[*]`
109+
* `./test_pytest.py::TestParam::test_param_13[*]`
110+
* `./test_pytest.py::TestParamAll::test_param_13[*]`
111+
* `./test_pytest.py::TestParamAll::test_spam_13[*]`
112+
* `./test_pytest.py::test_fixture_param[*]`
113+
* `./test_pytest.py::test_param_fixture[*]`
114+
* `./test_pytest_param.py::test_param_13[*]`
115+
* `./test_pytest_param.py::TestParamAll::test_param_13[*]`
116+
* `./test_pytest_param.py::TestParamAll::test_spam_13[*]`
117+
* (`./test_unittest.py::MyTests::test_with_subtests`)
118+
* (`./test_unittest.py::MyTests::test_with_nested_subtests`)
119+
* (`./test_unittest.py::MyTests::test_dynamic_*`)
120+
121+
For more options for pytests's parametrize(), see
122+
https://docs.pytest.org/en/latest/example/parametrize.html#paramexamples.
123+
124+
using fixtures:
125+
126+
* `./test_pytest.py::test_fixture`
127+
* `./test_pytest.py::test_fixture_param[*]`
128+
* `./test_pytest.py::test_param_fixture[*]`
129+
* `./test_pytest.py::test_param_mark_fixture[*]`
130+
131+
other markers:
132+
133+
* `./test_pytest.py::test_known_failure`
134+
* `./test_pytest.py::test_param_markers[2]`
135+
* `./test_pytest.py::test_warned`
136+
* `./test_pytest.py::test_custom_marker`
137+
* `./test_pytest.py::test_multiple_markers`
138+
* (`./test_unittest.py::MyTests::test_known_failure`)
139+
140+
others not discovered:
141+
142+
* (`./test_pytest.py::TestSpam::TestHam::TestEggs::TestNoop1`)
143+
* (`./test_pytest.py::TestSpam::TestNoop2`)
144+
* (`./test_pytest.py::TestNoop3`)
145+
* (`./test_pytest.py::MyTests::test_simple`)
146+
* (`./test_unittest.py::MyTests::TestSub1`)
147+
* (`./test_unittest.py::MyTests::TestSub2`)
148+
* (`./test_unittest.py::NoTests`)
149+
150+
doctests:
151+
152+
* `./test_doctest.txt::test_doctest.txt`
153+
* (`./test_doctest.py::test_doctest.py`)
154+
* (`../mod.py::mod`)
155+
* (`../mod.py::mod.square`)
156+
* (`../mod.py::mod.Spam`)
157+
* (`../mod.py::mod.spam.eggs`)

0 commit comments

Comments
 (0)