Skip to content

Commit 020bc56

Browse files
ericsnowcurrentlyericsnowcurrently
authored
Add a Python script that uses the PyTest API to run test discovery. (microsoft#4605)
(for microsoft#4033)
1 parent 942cd9a commit 020bc56

17 files changed

Lines changed: 1150 additions & 1 deletion

File tree

‎build/ci/templates/compile-and-validate.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,10 @@ jobs:
156156
arguments: '-m pip install --upgrade -r ./build/test-requirements.txt'
157157

158158

159+
- bash: 'python -m pythonFiles.tests'
160+
displayName: 'run pythonFiles unit tests'
161+
162+
159163
- task: CmdLine@1
160164
displayName: 'pip install python packages'
161165
inputs:

‎news/3 Code Health/4033.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add a Python script to run PyTest correctly for discovery.

‎pythonFiles/testing_tools/adapter/__init__.py‎

Whitespace-only changes.

‎pythonFiles/testing_tools/adapter/__main__.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
from __future__ import absolute_import
2+
3+
import argparse
4+
import sys
5+
6+
from . import pytest, report
7+
from .errors import UnsupportedToolError, UnsupportedCommandError
8+
9+
10+
# Set this to True to pretty-print the output.
11+
DEBUG=False
12+
#DEBUG=True
13+
14+
TOOLS = {
15+
'pytest': {
16+
'_add_subparser': pytest.add_cli_subparser,
17+
'discover': pytest.discover,
18+
},
19+
}
20+
REPORTERS = {
21+
'discover': report.report_discovered,
22+
}
23+
24+
25+
def parse_args(
26+
argv=sys.argv[1:],
27+
prog=sys.argv[0],
28+
):
29+
"""
30+
Return the subcommand & tool to run, along with its args.
31+
32+
This defines the standard CLI for the different testing frameworks.
33+
"""
34+
parser = argparse.ArgumentParser(
35+
description='Run Python testing operations.',
36+
prog=prog,
37+
)
38+
cmdsubs = parser.add_subparsers(dest='cmd')
39+
40+
# Add "run" and "debug" subcommands when ready.
41+
for cmdname in ['discover']:
42+
sub = cmdsubs.add_parser(cmdname)
43+
subsubs = sub.add_subparsers(dest='tool')
44+
for toolname in sorted(TOOLS):
45+
try:
46+
add_subparser = TOOLS[toolname]['_add_subparser']
47+
except KeyError:
48+
continue
49+
add_subparser(cmdname, toolname, subsubs)
50+
51+
# Parse the args!
52+
args, toolargs = parser.parse_known_args(argv)
53+
ns = vars(args)
54+
55+
cmd = ns.pop('cmd')
56+
if not cmd:
57+
parser.error('missing command')
58+
tool = ns.pop('tool')
59+
if not tool:
60+
parser.error('missing tool')
61+
62+
return tool, cmd, ns, toolargs
63+
64+
65+
def main(toolname, cmdname, subargs, toolargs,
66+
_tools=TOOLS, _reporters=REPORTERS):
67+
try:
68+
tool = _tools[toolname]
69+
except KeyError:
70+
raise UnsupportedToolError(toolname)
71+
72+
try:
73+
run = tool[cmdname]
74+
report_result = _reporters[cmdname]
75+
except KeyError:
76+
raise UnsupportedCommandError(cmdname)
77+
78+
result = run(toolargs, **subargs)
79+
report_result(result, debug=DEBUG)
80+
81+
82+
if __name__ == '__main__':
83+
tool, cmd, subargs, toolargs = parse_args()
84+
main(tool, cmd, subargs, toolargs)

‎pythonFiles/testing_tools/adapter/errors.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
2+
class UnsupportedToolError(ValueError):
3+
def __init__(self, tool):
4+
super().__init__('unsupported tool {!r}'.format(tool))
5+
self.tool = tool
6+
7+
8+
class UnsupportedCommandError(ValueError):
9+
def __init__(self, cmd):
10+
super().__init__('unsupported cmd {!r}'.format(cmd))
11+
self.cmd = cmd

‎pythonFiles/testing_tools/adapter/info.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from collections import namedtuple
2+
3+
4+
class TestPath(namedtuple('TestPath', 'root relfile func sub')):
5+
"""Where to find a single test."""
6+
7+
8+
class TestInfo(namedtuple('TestInfo', 'id name path lineno markers')):
9+
"""Info for a single test."""

‎pythonFiles/testing_tools/adapter/pytest.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import os.path
2+
3+
import pytest
4+
5+
from .errors import UnsupportedCommandError
6+
from .info import TestInfo, TestPath
7+
8+
9+
def add_cli_subparser(cmd, name, parent):
10+
"""Add a new subparser to the given parent and add args to it."""
11+
parser = parent.add_parser(name)
12+
if cmd == 'discover':
13+
# For now we don't have any tool-specific CLI options to add.
14+
pass
15+
else:
16+
raise UnsupportedCommandError(cmd)
17+
return parser
18+
19+
20+
def discover(pytestargs=None,
21+
_pytest_main=pytest.main, _plugin=None):
22+
"""Return the results of test discovery."""
23+
if _plugin is None:
24+
_plugin = TestCollector()
25+
26+
pytestargs = _adjust_pytest_args(pytestargs)
27+
ec = _pytest_main(pytestargs, [_plugin])
28+
if ec != 0:
29+
raise Exception('pytest discovery failed (exit code {})'.format(ec))
30+
if _plugin.discovered is None:
31+
raise Exception('pytest discovery did not start')
32+
return _plugin.discovered
33+
34+
35+
def _adjust_pytest_args(pytestargs):
36+
pytestargs = list(pytestargs) if pytestargs else []
37+
# Duplicate entries should be okay.
38+
pytestargs.insert(0, '--collect-only')
39+
pytestargs.insert(0, '-pno:terminal')
40+
# TODO: pull in code from:
41+
# src/client/unittests/pytest/services/discoveryService.ts
42+
# src/client/unittests/pytest/services/argsService.ts
43+
return pytestargs
44+
45+
46+
class TestCollector(object):
47+
"""This is a pytest plugin that collects the discovered tests."""
48+
49+
discovered = None
50+
51+
# Relevant plugin hooks:
52+
# https://docs.pytest.org/en/latest/reference.html#collection-hooks
53+
54+
def pytest_collection_modifyitems(self, session, config, items):
55+
self.discovered = []
56+
for item in items:
57+
info = _parse_item(item)
58+
self.discovered.append(info)
59+
60+
# This hook is not specified in the docs, so we also provide
61+
# the "modifyitems" hook just in case.
62+
def pytest_collection_finish(self, session):
63+
try:
64+
items = session.items
65+
except AttributeError:
66+
# TODO: Is there an alternative?
67+
return
68+
# print(', '.join(k for k in dir(items[0]) if k[0].islower()))
69+
self.discovered = []
70+
for item in items:
71+
# print(' ', item.user_properties)
72+
# print(' ', item.own_markers)
73+
# print(' ', list(item.iter_markers()))
74+
# print()
75+
info = _parse_item(item)
76+
self.discovered.append(info)
77+
78+
79+
def _parse_item(item):
80+
"""
81+
(pytest.Collector)
82+
pytest.Session
83+
pytest.Package
84+
pytest.Module
85+
pytest.Class
86+
(pytest.File)
87+
(pytest.Item)
88+
pytest.Function
89+
"""
90+
# Figure out the file.
91+
filename, lineno, fullname = item.location
92+
if not str(item.fspath).endswith(os.path.sep + filename):
93+
raise NotImplementedError
94+
testroot = str(item.fspath)[:-len(filename)].rstrip(os.path.sep)
95+
if os.path.sep in filename:
96+
relfile = filename
97+
else:
98+
relfile = os.path.join('.', filename)
99+
100+
# Figure out the func (and subs).
101+
funcname = item.function.__name__
102+
parts = item.nodeid.split('::')
103+
if parts.pop(0) != filename:
104+
# TODO: What to do?
105+
raise NotImplementedError
106+
suites = []
107+
while len(parts) > 1:
108+
suites.append(parts.pop(0))
109+
parameterized = ''
110+
if '[' in parts[0]:
111+
_func, sep, parameterized = parts[0].partition('[')
112+
parameterized = sep + parameterized
113+
if _func != funcname:
114+
# TODO: What to do?
115+
raise NotImplementedError
116+
if suites:
117+
testfunc = '.'.join(suites) + '.' + funcname
118+
else:
119+
testfunc = funcname
120+
if fullname != testfunc + parameterized:
121+
# TODO: What to do?
122+
raise NotImplementedError
123+
124+
# Sort out markers.
125+
# See: https://docs.pytest.org/en/latest/reference.html#marks
126+
markers = set()
127+
for marker in item.own_markers:
128+
if marker.name == 'parameterize':
129+
# We've already covered these.
130+
continue
131+
elif marker.name == 'skip':
132+
markers.add('skip')
133+
elif marker.name == 'skipif':
134+
markers.add('skip-if')
135+
elif marker.name == 'xfail':
136+
markers.add('expected-failure')
137+
# TODO: Support other markers?
138+
139+
return TestInfo(
140+
id=item.nodeid,
141+
name=item.name,
142+
path=TestPath(
143+
root=testroot,
144+
relfile=relfile,
145+
func=testfunc,
146+
sub=[parameterized] if parameterized else None,
147+
),
148+
lineno=lineno,
149+
markers=sorted(markers) if markers else None,
150+
)

‎pythonFiles/testing_tools/adapter/report.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import json
2+
3+
4+
def report_discovered(tests, debug=False,
5+
_send=print):
6+
"""Serialize the discovered tests and write to stdout."""
7+
data = [{
8+
'id': test.id,
9+
'name': test.name,
10+
'testroot': test.path.root,
11+
'relfile': test.path.relfile,
12+
'lineno': test.lineno,
13+
'testfunc': test.path.func,
14+
'subtest': test.path.sub or None,
15+
'markers': test.markers or None,
16+
} for test in tests]
17+
kwargs = {}
18+
if debug:
19+
# human-formatted
20+
kwargs = dict(
21+
sort_keys=True,
22+
indent=4,
23+
separators=(',', ': '),
24+
)
25+
serialized = json.dumps(data, **kwargs)
26+
_send(serialized)

‎pythonFiles/testing_tools/run_adapter.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Replace the "." entry.
2+
import os.path
3+
import sys
4+
sys.path[0] = os.path.dirname(
5+
os.path.dirname(
6+
os.path.abspath(__file__)))
7+
8+
from testing_tools.adapter.__main__ import parse_args, main
9+
10+
11+
if __name__ == '__main__':
12+
tool, cmd, subargs, toolargs = parse_args()
13+
main(tool, cmd, subargs, toolargs)

pythonFiles/tests/run_all.py renamed to pythonFiles/tests/__main__.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77
TEST_ROOT = os.path.dirname(__file__)
88
SRC_ROOT = os.path.dirname(TEST_ROOT)
99
DATASCIENCE_ROOT = os.path.join(SRC_ROOT, 'datascience')
10+
TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, 'testing_tools')
1011

1112

1213
if __name__ == '__main__':
1314
sys.path.insert(1, DATASCIENCE_ROOT)
15+
sys.path.insert(1, TESTING_TOOLS_ROOT)
1416
ec = pytest.main([
1517
'--rootdir', SRC_ROOT,
1618
TEST_ROOT,
17-
])
19+
] + sys.argv[1:])
1820
sys.exit(ec)

0 commit comments

Comments
 (0)