Skip to content

Commit e69d473

Browse files
tamarinvs19tamarinvs19
committed
First version of instruction coverage
1 parent be47d83 commit e69d473

File tree

13 files changed

+192
-56
lines changed

13 files changed

+192
-56
lines changed

‎utbot-python-executor/src/main/python/utbot_executor/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "utbot-executor"
3-
version = "1.7.0"
3+
version = "1.7.0.dev7"
44
description = ""
55
authors = ["Vyacheslav Tamarin <vyacheslav.tamarin@yandex.ru>"]
66
readme = "README.md"

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Python code executor for UnitTestBot"""
22
import copy
3+
import dis
34
import importlib
4-
import inspect
55
import logging
66
import pathlib
77
import socket
@@ -17,8 +17,8 @@
1717
from utbot_executor.deep_serialization.utils import PythonId, getattr_by_path
1818
from utbot_executor.memory_compressor import compress_memory
1919
from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse
20-
from utbot_executor.ut_tracer import UtTracer
21-
from utbot_executor.utils import suppress_stdout as __suppress_stdout
20+
from utbot_executor.ut_tracer import UtTracer, UtCoverageSender
21+
from utbot_executor.utils import suppress_stdout as __suppress_stdout, get_instructions
2222

2323
__all__ = ['PythonExecutor']
2424

@@ -111,22 +111,23 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse:
111111
state_init = _update_states(loader.reload_id(), state_init_memory)
112112
serialized_state_init = serialize_memory_dump(state_init)
113113

114-
def _coverage_sender(info: typing.Tuple[str, int]):
115-
if pathlib.Path(info[0]) == pathlib.Path(request.filepath):
116-
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
117-
logging.debug("Coverage message: %s:%d", request.coverage_id, info[1])
118-
logging.debug("Port: %d", self.coverage_port)
119-
message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8')
120-
sock.sendto(message, (self.coverage_hostname, self.coverage_port))
121-
logging.debug("ID: %s, Coverage: %s", request.coverage_id, info)
114+
# def _coverage_sender(info: typing.Tuple[str, int]):
115+
# if pathlib.Path(info[0]) == pathlib.Path(request.filepath):
116+
# sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
117+
# logging.debug("Coverage message: %s:%d", request.coverage_id, info[1])
118+
# logging.debug("Port: %d", self.coverage_port)
119+
# message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8')
120+
# sock.sendto(message, (self.coverage_hostname, self.coverage_port))
121+
# logging.debug("ID: %s, Coverage: %s", request.coverage_id, info)
122+
_coverage_sender = UtCoverageSender(request.coverage_id, self.coverage_hostname, self.coverage_port)
122123

123124
value = _run_calculate_function_value(
124125
function,
125126
args,
126127
kwargs,
127128
request.filepath,
128129
serialized_state_init,
129-
tracer=UtTracer(_coverage_sender)
130+
tracer=UtTracer(pathlib.Path(request.filepath), [sys.prefix, sys.exec_prefix], _coverage_sender)
130131
)
131132
except Exception as _:
132133
logging.debug("Error \n%s", traceback.format_exc())
@@ -172,10 +173,10 @@ def _run_calculate_function_value(
172173

173174
__is_exception = False
174175

175-
(__sources, __start, ) = inspect.getsourcelines(function)
176-
__not_empty_lines = [i for i, line in enumerate(__sources, __start) if len(line.strip()) != 0]
177-
logging.debug("Not empty lines %s", __not_empty_lines)
178-
__end = __start + len(__sources)
176+
# bytecode = dis.get_instructions(function)
177+
# __all_code_lines = [instr.starts_line for instr in bytecode if instr.starts_line is not None]
178+
__all_code_lines = list(get_instructions(function))
179+
__start = min([op[0] for op in __all_code_lines])
179180

180181
__tracer = tracer
181182

@@ -189,14 +190,15 @@ def _run_calculate_function_value(
189190

190191
logging.debug("Coverage: %s", __tracer.counts)
191192
logging.debug("Fullpath: %s", fullpath)
192-
module_path = pathlib.Path(fullpath)
193-
__stmts = [x[1] for x in __tracer.counts if pathlib.Path(x[0]) == module_path]
194-
__stmts_filtered = [x for x in __not_empty_lines if x in __stmts]
195-
__stmts_filtered_with_def = [__start] + __stmts_filtered
196-
__missed_filtered = [x for x in __not_empty_lines if x not in __stmts_filtered_with_def]
197-
logging.debug("Covered lines: %s", __stmts_filtered_with_def)
193+
__stmts = [x for x in __tracer.counts]
194+
__stmts_with_def = [(1, 0)] + __stmts
195+
__missed_filtered = [x for x in __all_code_lines if x not in __stmts_with_def]
196+
logging.debug("Covered lines: %s", __stmts_with_def)
198197
logging.debug("Missed lines: %s", __missed_filtered)
199198

199+
__str_statements = [":".join(map(str, x)) for x in __stmts_with_def]
200+
__str_missed_statements = [":".join(map(str, x)) for x in __missed_filtered]
201+
200202
args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result)
201203
ids = args_ids + list(kwargs_ids.values())
202204
# state_before, state_after = compress_memory(ids, state_before, state_after)
@@ -205,8 +207,8 @@ def _run_calculate_function_value(
205207
return ExecutionSuccessResponse(
206208
status="success",
207209
is_exception=__is_exception,
208-
statements=__stmts_filtered_with_def,
209-
missed_statements=__missed_filtered,
210+
statements=__str_statements,
211+
missed_statements=__str_missed_statements,
210212
state_init=state_init,
211213
state_before=serialized_state_before,
212214
state_after=serialized_state_after,

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@ def handler(self) -> None:
8080
response_size = str(len(bytes_data))
8181
self.clientsocket.send((response_size + os.linesep).encode())
8282

83-
sended_size = 0
84-
while len(bytes_data) > sended_size:
85-
sended_size += self.clientsocket.send(bytes_data[sended_size:])
83+
sent_size = 0
84+
while len(bytes_data) > sent_size:
85+
sent_size += self.clientsocket.send(bytes_data[sent_size:])
8686

8787
logging.debug('Sent all data')
8888
logging.info('All done...')

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import dataclasses
22
import json
3-
from typing import Dict, List, Union
3+
from typing import Dict, List, Union, Tuple
44

55

66
@dataclasses.dataclass
@@ -24,8 +24,8 @@ class ExecutionResponse:
2424
class ExecutionSuccessResponse(ExecutionResponse):
2525
status: str
2626
is_exception: bool
27-
statements: List[int]
28-
missed_statements: List[int]
27+
statements: List[str]
28+
missed_statements: List[str]
2929
state_init: str
3030
state_before: str
3131
state_after: str

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
1+
import dis
2+
import inspect
3+
import logging
14
import os
5+
import pathlib
6+
import queue
7+
import socket
28
import sys
9+
import threading
310
import typing
411

512

@@ -9,12 +16,48 @@ def _modname(path):
916
return filename
1017

1118

19+
class UtCoverageSender:
20+
def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False):
21+
self.coverage_id = coverage_id
22+
self.host = host
23+
self.port = port
24+
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
25+
self.message_queue = queue.Queue()
26+
27+
self.use_thread = use_thread
28+
if use_thread:
29+
self.thread = threading.Thread(target=self.send_loop, daemon=True)
30+
self.thread.start()
31+
32+
def send_loop(self):
33+
try:
34+
while True:
35+
self.send_message_thread()
36+
except Exception as _:
37+
self.send_loop()
38+
39+
def send_message(self, message: bytes):
40+
self.sock.sendto(*message)
41+
42+
def send_message_thread(self):
43+
message = self.message_queue.get()
44+
self.send_message(message)
45+
46+
def put_message(self, key: str):
47+
message = bytes(f"{self.coverage_id}:{key}", encoding="utf-8")
48+
if self.use_thread:
49+
self.message_queue.put((message, (self.host, self.port)))
50+
else:
51+
self.send_message(message)
52+
53+
1254
class UtTracer:
13-
def __init__(self, sender: typing.Callable[[typing.Tuple[str, int]], None]):
14-
self.globaltrace = self.globaltrace_lt
55+
def __init__(self, tested_file: pathlib.Path, ignore_dirs: typing.List[str], sender: UtCoverageSender):
56+
self.tested_file = tested_file
1557
self.counts = {}
1658
self.localtrace = self.localtrace_count
1759
self.globaltrace = self.globaltrace_lt
60+
self.ignore_dirs = ignore_dirs
1861
self.sender = sender
1962

2063
def runfunc(self, func, /, *args, **kw):
@@ -31,22 +74,27 @@ def coverage(self, filename: str) -> typing.List[int]:
3174
return [line for file, line in self.counts.keys() if file == filename]
3275

3376
def localtrace_count(self, frame, why, arg):
34-
if why == "line":
35-
filename = frame.f_code.co_filename
77+
filename = frame.f_code.co_filename
78+
if pathlib.Path(filename) == self.tested_file:
3679
lineno = frame.f_lineno
37-
key = filename, lineno
80+
offset = 0
81+
if why == "opcode":
82+
offset = frame.f_lasti
83+
key = (lineno, offset)
3884
if key not in self.counts:
3985
try:
40-
self.sender(key)
86+
self.sender.put_message(":".join(map(str, key)))
4187
except Exception:
4288
pass
4389
self.counts[key] = self.counts.get(key, 0) + 1
4490
return self.localtrace
4591

4692
def globaltrace_lt(self, frame, why, arg):
4793
if why == 'call':
94+
frame.f_trace_opcodes = True
95+
frame.f_trace_lines = False
4896
filename = frame.f_globals.get('__file__', None)
49-
if filename:
97+
if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs):
5098
modulename = _modname(filename)
5199
if modulename is not None:
52100
return self.localtrace
@@ -60,3 +108,18 @@ def __init__(self):
60108

61109
def runfunc(self, func, /, *args, **kw):
62110
return func(*args, **kw)
111+
112+
113+
def f(x):
114+
if 0 < x < 10 and x % 2 == 0:
115+
return 1
116+
else:
117+
return [100,
118+
x**2,
119+
x + 1
120+
]
121+
122+
123+
if __name__ in "__main__":
124+
tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0))
125+
tracer.runfunc(f, 70)

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1+
import dis
12
import os
23
import sys
4+
import typing
35
from contextlib import contextmanager
46

57

@@ -12,3 +14,14 @@ def suppress_stdout():
1214
yield
1315
finally:
1416
sys.stdout = old_stdout
17+
18+
19+
def get_instructions(obj: object) -> typing.Iterator[tuple[int, int]]:
20+
def inner_get_instructions(x, current_line):
21+
for i, el in enumerate(dis.get_instructions(x)):
22+
if el.starts_line is not None:
23+
current_line = el.starts_line
24+
yield current_line, el.offset
25+
if "<class 'code'>" in str(type(el.argval)):
26+
inner_get_instructions(el.argval, current_line)
27+
return inner_get_instructions(obj, None)

‎utbot-python-executor/src/main/resources/utbot_executor_version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.7.0
1+
1.7.0.dev7

‎utbot-python/samples/samples/controlflow/multi_conditions.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
def check_interval(x: float, left: float, right: float) -> str:
2+
if left < x < right or right < x < left:
3+
return "between"
4+
elif x < left and x < right:
5+
return "less"
6+
elif x > left and x > right:
7+
return "more"
8+
elif left == right:
9+
return "all equals"
10+
elif x == left:
11+
return "left"
12+
elif x == right:
13+
return "right"
14+
return "what?"

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import org.utbot.python.evaluation.*
1212
import org.utbot.python.evaluation.serialization.MemoryDump
1313
import org.utbot.python.evaluation.serialization.toPythonTree
1414
import org.utbot.python.evaluation.utils.CoverageIdGenerator
15+
import org.utbot.python.evaluation.utils.PyInstruction
1516
import org.utbot.python.evaluation.utils.coveredLinesToInstructions
17+
import org.utbot.python.evaluation.utils.makeInstructions
1618
import org.utbot.python.framework.api.python.PythonTree
1719
import org.utbot.python.framework.api.python.PythonTreeModel
1820
import org.utbot.python.framework.api.python.PythonTreeWrapper
@@ -92,7 +94,7 @@ class PythonEngine(
9294
private fun handleTimeoutResult(
9395
arguments: List<PythonFuzzedValue>,
9496
methodUnderTestDescription: PythonMethodDescription,
95-
coveredLines: Collection<Int>,
97+
coveredInstructions: List<Instruction>,
9698
): FuzzingExecutionFeedback {
9799
val summary = arguments
98100
.zip(methodUnderTest.arguments)
@@ -109,7 +111,6 @@ class PythonEngine(
109111
val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) }
110112
val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) }
111113

112-
val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest)
113114
val coverage = Coverage(coveredInstructions)
114115
val utFuzzedExecution = PythonUtExecution(
115116
stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null),
@@ -134,7 +135,8 @@ class PythonEngine(
134135
): FuzzingExecutionFeedback {
135136
val prohibitedExceptions = listOf(
136137
"builtins.AttributeError",
137-
"builtins.TypeError"
138+
"builtins.TypeError",
139+
"builtins.NotImplementedError",
138140
)
139141

140142
val summary = arguments
@@ -237,8 +239,8 @@ class PythonEngine(
237239
is PythonEvaluationTimeout -> {
238240
val coveredLines =
239241
manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableSetOf())
240-
val utTimeoutException = handleTimeoutResult(arguments, description, coveredLines)
241-
val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest)
242+
val coveredInstructions = makeInstructions(coveredLines, methodUnderTest)
243+
val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions)
242244
val trieNode: Trie.Node<Instruction> =
243245
if (coveredInstructions.isEmpty())
244246
Trie.emptyNode()

0 commit comments

Comments
 (0)