Skip to content

Commit 8c86417

Browse files
eleanorjboydeleanorjboyd
authored
Add dynamic result (#21466)
fixes, microsoft#21148 and microsoft#21149
1 parent 55cd2e0 commit 8c86417

File tree

14 files changed

+215
-145
lines changed

14 files changed

+215
-145
lines changed

‎pythonFiles/tests/pytestadapter/helpers.py‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
3+
34
import io
45
import json
56
import os
@@ -9,7 +10,7 @@
910
import sys
1011
import threading
1112
import uuid
12-
from typing import Any, Dict, List, Optional, Tuple
13+
from typing import Any, Dict, List, Optional, Tuple, Union
1314

1415
TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"
1516
from typing_extensions import TypedDict
@@ -72,21 +73,34 @@ def process_rpc_message(data: str) -> Tuple[Dict[str, Any], str]:
7273
str_stream: io.StringIO = io.StringIO(data)
7374

7475
length: int = 0
76+
7577
while True:
7678
line: str = str_stream.readline()
7779
if CONTENT_LENGTH.lower() in line.lower():
7880
length = int(line[len(CONTENT_LENGTH) :])
7981
break
82+
8083
if not line or line.isspace():
8184
raise ValueError("Header does not contain Content-Length")
85+
8286
while True:
8387
line: str = str_stream.readline()
8488
if not line or line.isspace():
8589
break
8690

8791
raw_json: str = str_stream.read(length)
88-
dict_json: Dict[str, Any] = json.loads(raw_json)
89-
return dict_json, str_stream.read()
92+
return json.loads(raw_json), str_stream.read()
93+
94+
95+
def process_rpc_json(data: str) -> List[Dict[str, Any]]:
96+
"""Process the JSON data which comes from the server which runs the pytest discovery."""
97+
json_messages = []
98+
remaining = data
99+
while remaining:
100+
json_data, remaining = process_rpc_message(remaining)
101+
json_messages.append(json_data)
102+
103+
return json_messages
90104

91105

92106
def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]:
@@ -110,53 +124,58 @@ def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]:
110124
"PYTHONPATH": os.fspath(pathlib.Path(__file__).parent.parent.parent),
111125
}
112126
)
127+
completed = threading.Event()
113128

114-
result: list = []
129+
result = []
115130
t1: threading.Thread = threading.Thread(
116-
target=_listen_on_socket, args=(listener, result)
131+
target=_listen_on_socket, args=(listener, result, completed)
117132
)
118133
t1.start()
119134

120135
t2 = threading.Thread(
121-
target=lambda proc_args, proc_env, proc_cwd: subprocess.run(
122-
proc_args, env=proc_env, cwd=proc_cwd
123-
),
124-
args=(process_args, env, TEST_DATA_PATH),
136+
target=_run_test_code,
137+
args=(process_args, env, TEST_DATA_PATH, completed),
125138
)
126139
t2.start()
127140

128141
t1.join()
129142
t2.join()
130143

131-
a = process_rpc_json(result[0])
132-
return a if result else None
144+
return process_rpc_json(result[0]) if result else None
133145

134146

135-
def process_rpc_json(data: str) -> List[Dict[str, Any]]:
136-
"""Process the JSON data which comes from the server which runs the pytest discovery."""
137-
json_messages = []
138-
remaining = data
139-
while remaining:
140-
json_data, remaining = process_rpc_message(remaining)
141-
json_messages.append(json_data)
142-
143-
return json_messages
144-
145-
146-
def _listen_on_socket(listener: socket.socket, result: List[str]):
147+
def _listen_on_socket(
148+
listener: socket.socket, result: List[str], completed: threading.Event
149+
):
147150
"""Listen on the socket for the JSON data from the server.
148-
Created as a seperate function for clarity in threading.
151+
Created as a separate function for clarity in threading.
149152
"""
150153
sock, (other_host, other_port) = listener.accept()
154+
listener.settimeout(1)
151155
all_data: list = []
152156
while True:
153157
data: bytes = sock.recv(1024 * 1024)
154158
if not data:
155-
break
159+
if completed.is_set():
160+
break
161+
else:
162+
try:
163+
sock, (other_host, other_port) = listener.accept()
164+
except socket.timeout:
165+
result.append("".join(all_data))
166+
return
156167
all_data.append(data.decode("utf-8"))
157168
result.append("".join(all_data))
158169

159170

171+
def _run_test_code(
172+
proc_args: List[str], proc_env, proc_cwd: str, completed: threading.Event
173+
):
174+
result = subprocess.run(proc_args, env=proc_env, cwd=proc_cwd)
175+
completed.set()
176+
return result
177+
178+
160179
def find_test_line_number(test_name: str, test_file_path) -> str:
161180
"""Function which finds the correct line number for a test by looking for the "test_marker--[test_name]" string.
162181

‎pythonFiles/tests/pytestadapter/test_discovery.py‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,10 @@ def test_syntax_error(tmp_path):
5959
temp_dir.mkdir()
6060
p = temp_dir / "error_syntax_discovery.py"
6161
shutil.copyfile(file_path, p)
62-
actual_list: Optional[List[Dict[str, Any]]] = runner(
63-
["--collect-only", os.fspath(p)]
64-
)
65-
assert actual_list
66-
for actual in actual_list:
62+
actual = runner(["--collect-only", os.fspath(p)])
63+
if actual:
64+
actual = actual[0]
65+
assert actual
6766
assert all(item in actual for item in ("status", "cwd", "error"))
6867
assert actual["status"] == "error"
6968
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
@@ -76,11 +75,9 @@ def test_parameterized_error_collect():
7675
The json should still be returned but the errors list should be present.
7776
"""
7877
file_path_str = "error_parametrize_discovery.py"
79-
actual_list: Optional[List[Dict[str, Any]]] = runner(
80-
["--collect-only", file_path_str]
81-
)
82-
assert actual_list
83-
for actual in actual_list:
78+
actual = runner(["--collect-only", file_path_str])
79+
if actual:
80+
actual = actual[0]
8481
assert all(item in actual for item in ("status", "cwd", "error"))
8582
assert actual["status"] == "error"
8683
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
@@ -135,14 +132,15 @@ def test_pytest_collect(file, expected_const):
135132
file -- a string with the file or folder to run pytest discovery on.
136133
expected_const -- the expected output from running pytest discovery on the file.
137134
"""
138-
actual_list: Optional[List[Dict[str, Any]]] = runner(
135+
actual = runner(
139136
[
140137
"--collect-only",
141138
os.fspath(TEST_DATA_PATH / file),
142139
]
143140
)
144-
assert actual_list
145-
for actual in actual_list:
141+
if actual:
142+
actual = actual[0]
143+
assert actual
146144
assert all(item in actual for item in ("status", "cwd", "tests"))
147145
assert actual["status"] == "success"
148146
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)

‎pythonFiles/tests/pytestadapter/test_execution.py‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
# Licensed under the MIT License.
33
import os
44
import shutil
5-
from typing import Any, Dict, List, Optional
65

76
import pytest
87
from tests.pytestadapter import expected_execution_test_output
@@ -29,11 +28,10 @@ def test_syntax_error_execution(tmp_path):
2928
temp_dir.mkdir()
3029
p = temp_dir / "error_syntax_discovery.py"
3130
shutil.copyfile(file_path, p)
32-
actual_list: Optional[List[Dict[str, Any]]] = runner(
33-
["error_syntax_discover.py::test_function"]
34-
)
35-
assert actual_list
36-
for actual in actual_list:
31+
actual = runner(["error_syntax_discover.py::test_function"])
32+
if actual:
33+
actual = actual[0]
34+
assert actual
3735
assert all(item in actual for item in ("status", "cwd", "error"))
3836
assert actual["status"] == "error"
3937
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
@@ -45,9 +43,10 @@ def test_bad_id_error_execution():
4543
4644
The json should still be returned but the errors list should be present.
4745
"""
48-
actual_list: Optional[List[Dict[str, Any]]] = runner(["not/a/real::test_id"])
49-
assert actual_list
50-
for actual in actual_list:
46+
actual = runner(["not/a/real::test_id"])
47+
if actual:
48+
actual = actual[0]
49+
assert actual
5150
assert all(item in actual for item in ("status", "cwd", "error"))
5251
assert actual["status"] == "error"
5352
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
@@ -156,14 +155,17 @@ def test_pytest_execution(test_ids, expected_const):
156155
expected_const -- a dictionary of the expected output from running pytest discovery on the files.
157156
"""
158157
args = test_ids
159-
actual_list: Optional[List[Dict[str, Any]]] = runner(args)
160-
assert actual_list
161-
for actual in actual_list:
162-
assert all(item in actual for item in ("status", "cwd", "result"))
163-
assert actual["status"] == "success"
164-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
165-
result_data = actual["result"]
166-
for key in result_data:
167-
if result_data[key]["outcome"] == "failure":
168-
result_data[key]["message"] = "ERROR MESSAGE"
169-
assert result_data == expected_const
158+
actual = runner(args)
159+
assert actual
160+
print(actual)
161+
assert len(actual) == len(expected_const)
162+
actual_result_dict = dict()
163+
for a in actual:
164+
assert all(item in a for item in ("status", "cwd", "result"))
165+
assert a["status"] == "success"
166+
assert a["cwd"] == os.fspath(TEST_DATA_PATH)
167+
actual_result_dict.update(a["result"])
168+
for key in actual_result_dict:
169+
if actual_result_dict[key]["outcome"] == "failure":
170+
actual_result_dict[key]["message"] = "ERROR MESSAGE"
171+
assert actual_result_dict == expected_const

‎pythonFiles/tests/unittestadapter/test_execution.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,14 @@
33

44
import os
55
import pathlib
6+
import sys
67
from typing import List
78

89
import pytest
10+
11+
PYTHON_FILES = pathlib.Path(__file__).parent.parent
12+
13+
sys.path.insert(0, os.fspath(PYTHON_FILES))
914
from unittestadapter.execution import parse_execution_cli_args, run_tests
1015

1116
TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"

‎pythonFiles/unittestadapter/discovery.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
from typing_extensions import Literal
1818

1919
# Add the path to pythonFiles to sys.path to find testing_tools.socket_manager.
20-
PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21-
sys.path.insert(0, PYTHON_FILES)
20+
PYTHON_FILES = pathlib.Path(__file__).parent.parent
21+
sys.path.insert(0, os.fspath(PYTHON_FILES))
2222

2323
from testing_tools import socket_manager
2424

‎pythonFiles/unittestadapter/execution.py‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,14 @@
1313
from types import TracebackType
1414
from typing import Dict, List, Optional, Tuple, Type, Union
1515

16-
script_dir = pathlib.Path(__file__).parent.parent
17-
sys.path.append(os.fspath(script_dir))
18-
sys.path.append(os.fspath(script_dir / "lib" / "python"))
19-
from testing_tools import process_json_util
20-
16+
directory_path = pathlib.Path(__file__).parent.parent / "lib" / "python"
2117
# Add the path to pythonFiles to sys.path to find testing_tools.socket_manager.
22-
PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
23-
sys.path.insert(0, PYTHON_FILES)
18+
PYTHON_FILES = pathlib.Path(__file__).parent.parent
19+
20+
sys.path.insert(0, os.fspath(PYTHON_FILES))
2421
# Add the lib path to sys.path to find the typing_extensions module.
2522
sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python"))
26-
from testing_tools import socket_manager
23+
from testing_tools import process_json_util, socket_manager
2724
from typing_extensions import NotRequired, TypeAlias, TypedDict
2825
from unittestadapter.utils import parse_unittest_args
2926

@@ -54,6 +51,9 @@ def parse_execution_cli_args(
5451
ErrorType = Union[
5552
Tuple[Type[BaseException], BaseException, TracebackType], Tuple[None, None, None]
5653
]
54+
PORT = 0
55+
UUID = 0
56+
START_DIR = ""
5757

5858

5959
class TestOutcomeEnum(str, enum.Enum):
@@ -148,8 +148,10 @@ def formatResult(
148148
"traceback": tb,
149149
"subtest": subtest.id() if subtest else None,
150150
}
151-
152151
self.formatted[test_id] = result
152+
if PORT == 0 or UUID == 0:
153+
print("Error sending response, port or uuid unknown to python server.")
154+
send_run_data(result, PORT, UUID)
153155

154156

155157
class TestExecutionStatus(str, enum.Enum):
@@ -225,6 +227,33 @@ def run_tests(
225227
return payload
226228

227229

230+
def send_run_data(raw_data, port, uuid):
231+
# Build the request data (it has to be a POST request or the Node side will not process it), and send it.
232+
status = raw_data["outcome"]
233+
cwd = os.path.abspath(START_DIR)
234+
if raw_data["subtest"]:
235+
test_id = raw_data["subtest"]
236+
else:
237+
test_id = raw_data["test"]
238+
test_dict = {}
239+
test_dict[test_id] = raw_data
240+
payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict}
241+
addr = ("localhost", port)
242+
data = json.dumps(payload)
243+
request = f"""Content-Length: {len(data)}
244+
Content-Type: application/json
245+
Request-uuid: {uuid}
246+
247+
{data}"""
248+
try:
249+
with socket_manager.SocketManager(addr) as s:
250+
if s.socket is not None:
251+
s.socket.sendall(request.encode("utf-8"))
252+
except Exception as e:
253+
print(f"Error sending response: {e}")
254+
print(f"Request data: {request}")
255+
256+
228257
if __name__ == "__main__":
229258
# Get unittest test execution arguments.
230259
argv = sys.argv[1:]
@@ -270,11 +299,11 @@ def run_tests(
270299
print(f"Error: Could not connect to runTestIdsPort: {e}")
271300
print("Error: Could not connect to runTestIdsPort")
272301

273-
port, uuid = parse_execution_cli_args(argv[:index])
302+
PORT, UUID = parse_execution_cli_args(argv[:index])
274303
if test_ids_from_buffer:
275304
# Perform test execution.
276305
payload = run_tests(
277-
start_dir, test_ids_from_buffer, pattern, top_level_dir, uuid
306+
start_dir, test_ids_from_buffer, pattern, top_level_dir, UUID
278307
)
279308
else:
280309
cwd = os.path.abspath(start_dir)
@@ -284,19 +313,3 @@ def run_tests(
284313
"status": status,
285314
"error": "No test ids received from buffer",
286315
}
287-
288-
# Build the request data and send it.
289-
addr = ("localhost", port)
290-
data = json.dumps(payload)
291-
request = f"""Content-Length: {len(data)}
292-
Content-Type: application/json
293-
Request-uuid: {uuid}
294-
295-
{data}"""
296-
try:
297-
with socket_manager.SocketManager(addr) as s:
298-
if s.socket is not None:
299-
s.socket.sendall(request.encode("utf-8"))
300-
except Exception as e:
301-
print(f"Error sending response: {e}")
302-
print(f"Request data: {request}")

0 commit comments

Comments
 (0)