Skip to content

Use symbolic result instead of concrete if it is better #2700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EgorkaKulikov merged 3 commits into from
Nov 28, 2023
Merged

Use symbolic result instead of concrete if it is better #2700

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2a96ca5
Process exceptions better
EgorkaKulikov Nov 27, 2023
8d0a7c8
Cosmetic corrections
EgorkaKulikov Nov 27, 2023
bf506f9
Comment added
EgorkaKulikov Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Cosmetic corrections
  • Loading branch information
@EgorkaKulikov
EgorkaKulikov committed Nov 27, 2023
commit 8d0a7c86d01a9835a3a91c37fd5777b1ac61397a
124 changes: 53 additions & 71 deletions ...unit-contest/src/main/kotlin/org/utbot/contest/usvm/converter/JcToUtExecutionConverter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import org.usvm.instrumentation.util.enclosingMethod
import org.utbot.common.isPublic
import org.utbot.contest.usvm.jc.JcExecution
import org.utbot.contest.usvm.jc.UTestConcreteExecutionResult
import org.utbot.contest.usvm.jc.UTestResult
import org.utbot.contest.usvm.jc.UTestResultWrapper
import org.utbot.contest.usvm.jc.UTestSymbolicExceptionResult
import org.utbot.contest.usvm.jc.UTestSymbolicSuccessResult
import org.utbot.framework.codegen.domain.builtin.UtilMethodProvider
Expand Down Expand Up @@ -68,33 +68,34 @@ class JcToUtExecutionConverter(
private var uTestProcessResult = instToModelConverter.processUTest()

fun convert(): UtExecution? {
val coverage = convertCoverage(getTrace(jcExecution.uTestExecutionResult), jcExecution.method.enclosingType.jcClass)
val coverage = convertCoverage(getTrace(jcExecution.uTestExecutionResultWrapper), jcExecution.method.enclosingType.jcClass)

val utUsvmExecution: UtUsvmExecution = when (jcExecution.uTestExecutionResult) {
val utUsvmExecution: UtUsvmExecution = when (jcExecution.uTestExecutionResultWrapper) {
is UTestSymbolicExceptionResult -> {
UtUsvmExecution(
stateBefore = constructStateBeforeFromUTest(),
stateAfter = MissingState,
result = createExecutionFailureResult(
UTestExceptionDescriptor(
type = jcExecution.uTestExecutionResult.exceptionType,
exceptionDescriptor = UTestExceptionDescriptor(
type = jcExecution.uTestExecutionResultWrapper.exceptionType,
message = "",
stackTrace = emptyList(),
raisedByUserCode = true
raisedByUserCode = true,
),
jcExecution.method,
jcTypedMethod = jcExecution.method,
),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)
}

is UTestSymbolicSuccessResult -> {
val resultUtModel = jcExecution.uTestExecutionResult.let { res ->
res.initStatements.forEach { instToModelConverter.processInst(it) }
instToModelConverter.processInst(res.result)
instToModelConverter.findModelByInst(res.result)
}
val resultWrapper = jcExecution.uTestExecutionResultWrapper
resultWrapper.initStatements.forEach { instToModelConverter.processInst(it) }
instToModelConverter.processInst(resultWrapper.result)

val resultUtModel = instToModelConverter.findModelByInst(resultWrapper.result)

UtUsvmExecution(
stateBefore = constructStateBeforeFromUTest(),
stateAfter = MissingState,
Expand All @@ -104,73 +105,54 @@ class JcToUtExecutionConverter(
)
}

is UTestConcreteExecutionResult -> when (val executionResult =
jcExecution.uTestExecutionResult.uTestExecutionResult) {
is UTestExecutionSuccessResult -> UtUsvmExecution(
stateBefore = convertState(
executionResult.initialState,
EnvironmentStateKind.INITIAL,
jcExecution.method
),
stateAfter = convertState(
executionResult.resultState,
EnvironmentStateKind.FINAL,
jcExecution.method
),
// TODO usvm-sbft: ask why `UTestExecutionSuccessResult.result` is nullable
result = UtExecutionSuccess(executionResult.result?.let {
jcToUtModelConverter.convert(it, EnvironmentStateKind.FINAL)
} ?: UtVoidModel),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)

is UTestExecutionExceptionResult -> {
UtUsvmExecution(
stateBefore = convertState(
executionResult.initialState,
EnvironmentStateKind.INITIAL,
jcExecution.method
),
stateAfter = convertState(
executionResult.resultState,
EnvironmentStateKind.FINAL,
jcExecution.method
),
result = createExecutionFailureResult(
executionResult.cause,
jcExecution.method,
),
is UTestConcreteExecutionResult ->
when (val executionResult = jcExecution.uTestExecutionResultWrapper.uTestExecutionResult) {
is UTestExecutionSuccessResult -> UtUsvmExecution(
stateBefore = convertState(executionResult.initialState, EnvironmentStateKind.INITIAL, jcExecution.method),
stateAfter = convertState(executionResult.resultState, EnvironmentStateKind.FINAL, jcExecution.method),
// TODO usvm-sbft: ask why `UTestExecutionSuccessResult.result` is nullable
result = UtExecutionSuccess(executionResult.result?.let {
jcToUtModelConverter.convert(it, EnvironmentStateKind.FINAL)
} ?: UtVoidModel),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)
}

is UTestExecutionInitFailedResult -> {
logger.warn(convertException(executionResult.cause)) {
"Execution failed before method under test call on ${jcExecution.method.method}"
is UTestExecutionExceptionResult -> {
UtUsvmExecution(
stateBefore = convertState(executionResult.initialState, EnvironmentStateKind.INITIAL, jcExecution.method),
stateAfter = convertState(executionResult.resultState, EnvironmentStateKind.FINAL, jcExecution.method),
result = createExecutionFailureResult(executionResult.cause, jcExecution.method),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)
}
null
}

is UTestExecutionFailedResult -> {
logger.error(convertException(executionResult.cause)) {
"Concrete execution failed on ${jcExecution.method.method}"
is UTestExecutionInitFailedResult -> {
logger.warn(convertException(executionResult.cause)) {
"Execution failed before method under test call on ${jcExecution.method.method}"
}
null
}
null
}

is UTestExecutionTimedOutResult -> {
logger.warn { "Timeout on ${jcExecution.method.method}" }
UtUsvmExecution(
stateBefore = constructStateBeforeFromUTest(),
stateAfter = MissingState,
result = UtTimeoutException(TimeoutException("Concrete execution timed out")),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)
is UTestExecutionFailedResult -> {
logger.error(convertException(executionResult.cause)) {
"Concrete execution failed on ${jcExecution.method.method}"
}
null
}

is UTestExecutionTimedOutResult -> {
logger.warn { "Timeout on ${jcExecution.method.method}" }
UtUsvmExecution(
stateBefore = constructStateBeforeFromUTest(),
stateAfter = MissingState,
result = UtTimeoutException(TimeoutException("Concrete execution timed out")),
coverage = coverage,
instrumentation = uTestProcessResult.instrumentation,
)
}
}
}
} ?: return null

return utUsvmExecution
Expand Down Expand Up @@ -262,7 +244,7 @@ class JcToUtExecutionConverter(
cache = mutableMapOf()
)) as Throwable

private fun getTrace(executionResult: UTestResult): List<JcInst>? = when (executionResult) {
private fun getTrace(executionResult: UTestResultWrapper): List<JcInst>? = when (executionResult) {
is UTestConcreteExecutionResult -> when (val res = executionResult.uTestExecutionResult) {
is UTestExecutionExceptionResult -> res.trace
is UTestExecutionInitFailedResult -> res.trace
Expand Down
14 changes: 9 additions & 5 deletions utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcExecution.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,21 @@ import org.usvm.instrumentation.testcase.api.UTestInst
data class JcExecution(
val method: JcTypedMethod,
val uTest: UTest,
val uTestExecutionResult: UTestResult,
val uTestExecutionResultWrapper: UTestResultWrapper,
val coverage: JcCoverage
)

sealed interface UTestResult
sealed interface UTestResultWrapper

class UTestConcreteExecutionResult(val uTestExecutionResult: UTestExecutionResult) : UTestResult
class UTestConcreteExecutionResult(
val uTestExecutionResult: UTestExecutionResult
) : UTestResultWrapper

class UTestSymbolicExceptionResult(val exceptionType: JcType) : UTestResult
class UTestSymbolicExceptionResult(
val exceptionType: JcType
) : UTestResultWrapper

class UTestSymbolicSuccessResult(
val initStatements: List<UTestInst>,
val result: UTestExpression
) : UTestResult
) : UTestResultWrapper
13 changes: 5 additions & 8 deletions utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcTestExecutor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,30 +55,27 @@ class JcTestExecutor(
runner.executeAsync(uTest)
}

val result = if (execResult !is UTestExecutionSuccessResult && execResult !is UTestExecutionExceptionResult) {
// sometimes symbolic result is preferable that concolic: e.g. if concrete times out
val preferableResult =
if (execResult !is UTestExecutionSuccessResult && execResult !is UTestExecutionExceptionResult) {
val symbolicResult = state.methodResult
when (symbolicResult) {
is JcMethodResult.JcException -> {
UTestSymbolicExceptionResult(symbolicResult.type)
}

is JcMethodResult.JcException -> UTestSymbolicExceptionResult(symbolicResult.type)
is JcMethodResult.Success -> {
val resultScope = MemoryScope(ctx, model, state.memory, stringConstants, method)
val resultExpr = resultScope.resolveExpr(symbolicResult.value, method.returnType)
val resultInitializer = resultScope.decoderApi.initializerInstructions()
UTestSymbolicSuccessResult(resultInitializer, resultExpr)
}

JcMethodResult.NoCall -> UTestConcreteExecutionResult(execResult)
}

} else {
UTestConcreteExecutionResult(execResult)
}

val coverage = resolveCoverage(method, state)

return JcExecution(method, uTest, result, coverage)
return JcExecution(method, uTest, preferableResult, coverage)
}

@Suppress("UNUSED_PARAMETER")
Expand Down