Skip to content

Commit cf67b46

Browse files
tamarinvs19tamarinvs19
committed
Add instructions/lines trace mode, update coverage output
1 parent c933a79 commit cf67b46

File tree

17 files changed

+337
-183
lines changed

17 files changed

+337
-183
lines changed

‎utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,15 @@ import org.parsers.python.PythonParser
1010
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
1111
import org.utbot.framework.codegen.domain.TestFramework
1212
import org.utbot.framework.plugin.api.UtExecutionSuccess
13+
import org.utbot.python.evaluation.coverage.CoverageOutputFormat
1314
import org.utbot.python.PythonMethodHeader
1415
import org.utbot.python.PythonTestGenerationConfig
1516
import org.utbot.python.PythonTestSet
16-
import org.utbot.python.utils.RequirementsInstaller
1717
import org.utbot.python.TestFileInformation
18+
import org.utbot.python.utils.RequirementsInstaller
1819
import org.utbot.python.code.PythonCode
19-
import org.utbot.python.evaluation.utils.PythonCoverageMode
20-
import org.utbot.python.evaluation.utils.toPythonCoverageMode
20+
import org.utbot.python.evaluation.coverage.PythonCoverageMode
21+
import org.utbot.python.evaluation.coverage.toPythonCoverageMode
2122
import org.utbot.python.framework.api.python.PythonClassId
2223
import org.utbot.python.framework.codegen.model.Pytest
2324
import org.utbot.python.framework.codegen.model.Unittest
@@ -116,16 +117,20 @@ class PythonGenerateTestsCommand : CliktCommand(
116117
.choice("PASS", "FAIL")
117118
.default("FAIL")
118119

119-
private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite")
120+
private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite.")
120121
.flag(default = false)
121122

122-
private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement")
123+
private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement.")
123124
.choice("INSTRUCTIONS", "LINES")
124125
.default("INSTRUCTIONS")
125126

126-
private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "")
127+
private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.")
127128
.flag(default = false)
128129

130+
private val coverageOutputFormat by option("--coverage-output-format", help = "Use Lines, Instructions or TopFrameInstructions.")
131+
.choice("Instructions", "Lines", "TopFrameInstructions")
132+
.default("Instructions")
133+
129134
private val testFramework: TestFramework
130135
get() =
131136
when (testFrameworkAsString) {
@@ -263,7 +268,8 @@ class PythonGenerateTestsCommand : CliktCommand(
263268
isCanceled = { false },
264269
runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour),
265270
coverageMeasureMode = coverageMeasureMode.toPythonCoverageMode() ?: PythonCoverageMode.Instructions,
266-
sendCoverageContinuously = !doNotSendCoverageContinuously
271+
sendCoverageContinuously = !doNotSendCoverageContinuously,
272+
coverageOutputFormat = CoverageOutputFormat.valueOf(coverageOutputFormat),
267273
)
268274

269275
val processor = PythonCliProcessor(

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
suppress_stdout as __suppress_stdout,
2121
get_instructions,
2222
filter_instructions,
23-
TraceMode,
23+
TraceMode, UtInstruction,
2424
)
2525

2626
__all__ = ['PythonExecutor']
@@ -181,7 +181,7 @@ def _run_calculate_function_value(
181181
__is_exception = False
182182

183183
_, __start = inspect.getsourcelines(function)
184-
__all_code_lines = filter_instructions(get_instructions(function, __start), tracer.mode)
184+
__all_code_stmts = filter_instructions(get_instructions(function, __start), tracer.mode)
185185

186186
__tracer = tracer
187187

@@ -195,14 +195,13 @@ def _run_calculate_function_value(
195195

196196
logging.debug("Coverage: %s", __tracer.counts)
197197
logging.debug("Fullpath: %s", fullpath)
198-
__stmts = [x for x in __tracer.counts]
199-
__stmts_with_def = [(__start, 0)] + __stmts
200-
__missed_filtered = [x for x in __all_code_lines if x not in __stmts_with_def]
198+
__stmts_with_def = [UtInstruction(__start, 0, 0)] + list(__tracer.counts.keys())
199+
__missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def]
201200
logging.debug("Covered lines: %s", __stmts_with_def)
202201
logging.debug("Missed lines: %s", __missed_filtered)
203202

204-
__str_statements = [":".join(map(str, x)) for x in __stmts_with_def]
205-
__str_missed_statements = [":".join(map(str, x)) for x in __missed_filtered]
203+
__str_statements = [x.serialize() for x in __stmts_with_def]
204+
__str_missed_statements = [x.serialize() for x in __missed_filtered]
206205

207206
args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result)
208207
ids = args_ids + list(kwargs_ids.values())

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import dis
2+
import inspect
13
import logging
24
import os
35
import pathlib
@@ -7,7 +9,7 @@
79
import typing
810
from concurrent.futures import ThreadPoolExecutor
911

10-
from utbot_executor.utils import TraceMode
12+
from utbot_executor.utils import TraceMode, UtInstruction
1113

1214

1315
def _modname(path):
@@ -55,6 +57,11 @@ def put_message(self, key: str):
5557
self.send_message(message)
5658

5759

60+
class PureSender(UtCoverageSender):
61+
def __init__(self):
62+
super().__init__("000000", "localhost", 0, use_thread=False, send_coverage=False)
63+
64+
5865
class UtTracer:
5966
def __init__(
6067
self,
@@ -64,20 +71,23 @@ def __init__(
6471
mode: TraceMode = TraceMode.Instructions,
6572
):
6673
self.tested_file = tested_file
67-
self.counts = {}
74+
self.counts: dict[UtInstruction, int] = {}
6875
self.localtrace = self.localtrace_count
6976
self.globaltrace = self.globaltrace_lt
7077
self.ignore_dirs = ignore_dirs
7178
self.sender = sender
7279
self.mode = mode
80+
self.depth = 0
7381

7482
def runfunc(self, func, /, *args, **kw):
7583
result = None
84+
self.depth = 0
7685
sys.settrace(self.globaltrace)
7786
try:
7887
result = func(*args, **kw)
7988
finally:
8089
sys.settrace(None)
90+
self.depth = 0
8191
return result
8292

8393
def coverage(self, filename: str) -> typing.List[int]:
@@ -91,9 +101,9 @@ def localtrace_count(self, frame, why, arg):
91101
offset = lineno * 2
92102
if why == "opcode":
93103
offset = frame.f_lasti
94-
key = (lineno, offset)
104+
key = UtInstruction(lineno, offset, self.depth)
95105
if key not in self.counts:
96-
message = ":".join(map(str, key))
106+
message = key.serialize()
97107
try:
98108
self.sender.put_message(message)
99109
except Exception:
@@ -102,11 +112,12 @@ def localtrace_count(self, frame, why, arg):
102112
return self.localtrace
103113

104114
def globaltrace_lt(self, frame, why, arg):
115+
self.depth += 1
105116
if why == 'call':
106117
if self.mode == TraceMode.Instructions:
107118
frame.f_trace_opcodes = True
108119
frame.f_trace_lines = False
109-
filename = frame.f_globals.get('__file__', None)
120+
filename = frame.f_code.co_filename
110121
if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs):
111122
modulename = _modname(filename)
112123
if modulename is not None:
@@ -121,3 +132,15 @@ def __init__(self):
121132

122133
def runfunc(self, func, /, *args, **kw):
123134
return func(*args, **kw)
135+
136+
137+
def f(x):
138+
def g(x):
139+
xs = [[j for j in range(i)] for i in range(10)]
140+
return x * 2
141+
return x * g(x) + 2
142+
143+
144+
if __name__ == "__main__":
145+
tracer = UtTracer(pathlib.Path(__file__), [], PureSender())
146+
tracer.runfunc(f, 2)

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import dataclasses
12
import dis
23
import enum
34
import os
@@ -11,6 +12,19 @@ class TraceMode(enum.Enum):
1112
Instructions = 2
1213

1314

15+
@dataclasses.dataclass
16+
class UtInstruction:
17+
line: int
18+
offset: int
19+
depth: int
20+
21+
def serialize(self) -> str:
22+
return ":".join(map(str, [self.line, self.offset, self.depth]))
23+
24+
def __hash__(self):
25+
return hash((self.line, self.offset, self.depth))
26+
27+
1428
@contextmanager
1529
def suppress_stdout():
1630
with open(os.devnull, "w") as devnull:
@@ -22,21 +36,24 @@ def suppress_stdout():
2236
sys.stdout = old_stdout
2337

2438

25-
def get_instructions(obj: object, start_line: int) -> typing.Iterator[tuple[int, int]]:
26-
def inner_get_instructions(x, current_line):
39+
def get_instructions(obj: object, start_line: int) -> typing.Iterator[UtInstruction]:
40+
def inner_get_instructions(x, current_line, depth):
2741
for i, el in enumerate(dis.get_instructions(x)):
2842
if el.starts_line is not None:
2943
current_line = el.starts_line
30-
yield current_line, el.offset
44+
yield UtInstruction(current_line, el.offset, depth)
3145
if any(t in str(type(el.argval)) for t in ["<class 'code'>"]):
32-
inner_get_instructions(el.argval, current_line)
33-
return inner_get_instructions(obj, start_line)
46+
inner_get_instructions(el.argval, current_line, depth + 1)
47+
return inner_get_instructions(obj, start_line, 0)
3448

3549

3650
def filter_instructions(
37-
instructions: typing.Iterable[tuple[int, int]],
51+
instructions: typing.Iterable[UtInstruction],
3852
mode: TraceMode = TraceMode.Instructions,
39-
) -> list[tuple[int, int]]:
53+
) -> list[UtInstruction]:
4054
if mode == TraceMode.Lines:
41-
return [(it, 2 * it) for it in {line for line, op in instructions}]
55+
unique_line_instructions: set[UtInstruction] = set()
56+
for it in instructions:
57+
unique_line_instructions.add(UtInstruction(it.line, it.line * 2, it.depth))
58+
return list({UtInstruction(it.line, 2 * it.line, it.depth) for it in instructions})
4259
return list(instructions)

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,21 @@ import org.utbot.fuzzing.Control
88
import org.utbot.fuzzing.NoSeedValueException
99
import org.utbot.fuzzing.fuzz
1010
import org.utbot.fuzzing.utils.Trie
11-
import org.utbot.python.evaluation.*
11+
import org.utbot.python.evaluation.EvaluationCache
12+
import org.utbot.python.evaluation.PythonCodeExecutor
13+
import org.utbot.python.evaluation.PythonCodeSocketExecutor
14+
import org.utbot.python.evaluation.PythonEvaluationError
15+
import org.utbot.python.evaluation.PythonEvaluationSuccess
16+
import org.utbot.python.evaluation.PythonEvaluationTimeout
17+
import org.utbot.python.evaluation.PythonWorker
18+
import org.utbot.python.evaluation.PythonWorkerManager
19+
import org.utbot.python.evaluation.coverage.CoverageIdGenerator
20+
import org.utbot.python.evaluation.coverage.PyInstruction
21+
import org.utbot.python.evaluation.coverage.PythonCoverageMode
22+
import org.utbot.python.evaluation.coverage.calculateCoverage
23+
import org.utbot.python.evaluation.coverage.makeInstructions
1224
import org.utbot.python.evaluation.serialization.MemoryDump
1325
import org.utbot.python.evaluation.serialization.toPythonTree
14-
import org.utbot.python.evaluation.utils.CoverageIdGenerator
15-
import org.utbot.python.evaluation.utils.PythonCoverageMode
16-
import org.utbot.python.evaluation.utils.makeInstructions
1726
import org.utbot.python.framework.api.python.PythonTree
1827
import org.utbot.python.framework.api.python.PythonTreeModel
1928
import org.utbot.python.framework.api.python.PythonTreeWrapper
@@ -95,7 +104,7 @@ class PythonEngine(
95104
private fun handleTimeoutResult(
96105
arguments: List<PythonFuzzedValue>,
97106
methodUnderTestDescription: PythonMethodDescription,
98-
coveredInstructions: List<Instruction>,
107+
coveredInstructions: List<PyInstruction>,
99108
): FuzzingExecutionFeedback {
100109
val summary = arguments
101110
.zip(methodUnderTest.arguments)
@@ -112,7 +121,7 @@ class PythonEngine(
112121
val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) }
113122
val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) }
114123

115-
val coverage = Coverage(coveredInstructions)
124+
val coverage = Coverage(makeInstructions(coveredInstructions, methodUnderTest))
116125
val utFuzzedExecution = PythonUtExecution(
117126
stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null),
118127
stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null),
@@ -176,7 +185,7 @@ class PythonEngine(
176185
stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null),
177186
diffIds = evaluationResult.diffIds,
178187
result = executionResult,
179-
coverage = evaluationResult.coverage,
188+
coverage = calculateCoverage(evaluationResult.coverage, methodUnderTest),
180189
testMethodName = testMethodName.testName?.camelToSnakeCase(),
181190
displayName = testMethodName.displayName,
182191
summary = summary.map { DocRegularStmt(it) },
@@ -238,11 +247,10 @@ class PythonEngine(
238247
}
239248

240249
is PythonEvaluationTimeout -> {
241-
val coveredLines =
250+
val coveredInstructions =
242251
manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf())
243-
val coveredInstructions = makeInstructions(coveredLines, methodUnderTest)
244252
val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions)
245-
val trieNode: Trie.Node<Instruction> =
253+
val trieNode: Trie.Node<PyInstruction> =
246254
if (coveredInstructions.isEmpty())
247255
Trie.emptyNode()
248256
else
@@ -266,7 +274,7 @@ class PythonEngine(
266274
val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback
267275
when (result) {
268276
is ValidExecution -> {
269-
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
277+
val trieNode: Trie.Node<PyInstruction> = description.tracer.add(coveredInstructions)
270278
description.limitManager.addSuccessExecution()
271279
PythonExecutionResult(
272280
result,
@@ -316,7 +324,7 @@ class PythonEngine(
316324
parameters,
317325
fuzzedConcreteValues,
318326
pythonTypeStorage,
319-
Trie(Instruction::id),
327+
Trie(PyInstruction::id),
320328
Random(0),
321329
TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true),
322330
methodUnderTest.definition.type,
@@ -352,7 +360,7 @@ class PythonEngine(
352360
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
353361
val mem = cache.get(pair)
354362
if (mem != null) {
355-
logger.debug("Repeat in fuzzing ${arguments.map {it.tree}}")
363+
logger.debug { "Repeat in fuzzing ${arguments.map {it.tree}}" }
356364
description.limitManager.addSuccessExecution()
357365
emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback))
358366
return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache()

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import org.utbot.framework.minimization.minimizeExecutions
66
import org.utbot.framework.plugin.api.UtError
77
import org.utbot.framework.plugin.api.UtExecution
88
import org.utbot.framework.plugin.api.UtExecutionSuccess
9-
import org.utbot.python.evaluation.utils.PythonCoverageMode
9+
import org.utbot.python.evaluation.coverage.PythonCoverageMode
1010
import org.utbot.python.framework.api.python.PythonUtExecution
1111
import org.utbot.python.framework.api.python.util.pythonStrClassId
1212
import org.utbot.python.fuzzing.*

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ package org.utbot.python
22

33
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
44
import org.utbot.framework.codegen.domain.TestFramework
5-
import org.utbot.python.evaluation.utils.PythonCoverageMode
5+
import org.utbot.python.evaluation.coverage.CoverageOutputFormat
6+
import org.utbot.python.evaluation.coverage.PythonCoverageMode
67
import java.nio.file.Path
78

89
data class TestFileInformation(
@@ -25,4 +26,5 @@ class PythonTestGenerationConfig(
2526
val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour,
2627
val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions,
2728
val sendCoverageContinuously: Boolean = true,
29+
val coverageOutputFormat: CoverageOutputFormat = CoverageOutputFormat.Lines,
2830
)

0 commit comments

Comments
 (0)