Skip to content

Summaries for tests generated by fuzzer are generated in the summary module #597 #599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amandelpie merged 15 commits into from
Aug 8, 2022
Merged

Summaries for tests generated by fuzzer are generated in the summary module #597 #599

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
ac224e6
Just moved some classes and shifted the stage of name generation for …
amandelpie Jul 27, 2022
10f1629
Merge branch 'main' into amandelpie/feature_597
amandelpie Aug 1, 2022
7ceeb4b
Made some refactoring
amandelpie Aug 1, 2022
010a8d1
Fixed a bug
amandelpie Aug 1, 2022
d111e5b
Changed signatures
amandelpie Aug 1, 2022
e9c62b7
Merge branch 'main' into amandelpie/feature_597
amandelpie Aug 5, 2022
5554705
Fixed merge issues
amandelpie Aug 5, 2022
46f8533
Add some fixes
amandelpie Aug 5, 2022
81e0759
Add some fixes
amandelpie Aug 5, 2022
707b6a1
Add some fixes
amandelpie Aug 5, 2022
fede3eb
Fixed some places
amandelpie Aug 8, 2022
0af780c
Fixed some places
amandelpie Aug 8, 2022
01d68e2
Fixed some places
amandelpie Aug 8, 2022
0722fed
Fixed wildcard imports
amandelpie Aug 8, 2022
1240c20
Merge branch 'main' into amandelpie/feature_597
amandelpie Aug 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'main' into amandelpie/feature_597 
# Conflicts:
#	utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
#	utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/context/CgContext.kt
#	utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt
#	utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt
#	utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt
#	utbot-framework/src/test/kotlin/org/utbot/examples/UtModelTestCaseChecker.kt
  • Loading branch information
@amandelpie
amandelpie committed Aug 5, 2022
commit e9c62b7339bf4cef72942585eef0a3b11b9f16ff
49 changes: 17 additions & 32 deletions utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ import org.utbot.framework.plugin.api.Step
import org.utbot.framework.plugin.api.UtAssembleModel
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
import org.utbot.framework.plugin.api.UtError
import org.utbot.framework.plugin.api.UtSymbolicExecution
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionCreator
import org.utbot.framework.plugin.api.UtInstrumentation
import org.utbot.framework.plugin.api.UtMethod
import org.utbot.framework.plugin.api.UtNullModel
Expand All @@ -82,6 +83,8 @@ import org.utbot.framework.plugin.api.util.description
import org.utbot.framework.util.jimpleBody
import org.utbot.framework.plugin.api.util.voidClassId
import org.utbot.fuzzer.*
import org.utbot.fuzzer.names.MethodBasedNameSuggester
import org.utbot.fuzzer.names.ModelBasedNameSuggester
import org.utbot.fuzzer.providers.ObjectModelProvider
import org.utbot.instrumentation.ConcreteExecutor
import soot.jimple.Stmt
Expand Down Expand Up @@ -287,14 +290,15 @@ class UtBotSymbolicEngine(
val concreteExecutionResult =
concreteExecutor.executeConcretely(methodUnderTest, stateBefore, instrumentation)

val concreteUtExecution = UtSymbolicExecution(
val concreteUtExecution = UtExecution(
stateBefore,
concreteExecutionResult.stateAfter,
concreteExecutionResult.result,
instrumentation,
mutableListOf(),
listOf(),
concreteExecutionResult.coverage
concreteExecutionResult.coverage,
UtExecutionCreator.SYMBOLIC_ENGINE
)
emit(concreteUtExecution)

Expand Down Expand Up @@ -471,23 +475,8 @@ class UtBotSymbolicEngine(
return@forEach
}
coveredInstructionValues[count] = values

emit(
UtFuzzedExecution(
stateBefore = initialEnvironmentModels,
stateAfter = concreteExecutionResult.stateAfter,
result = concreteExecutionResult.result,
coverage = concreteExecutionResult.coverage,
fuzzingValues = values,
fuzzedMethodDescription = methodUnderTestDescription
)
)
} catch (e: CancellationException) {
logger.debug { "Cancelled by timeout" }
} catch (e: ConcreteExecutionFailureException) {
emitFailedConcreteExecutionResult(initialEnvironmentModels, e)
} catch (e: Throwable) {
emit(UtError("Default concrete execution failed", e))
} else {
logger.error { "Coverage is empty for $methodUnderTest with ${values.map { it.model }}" }
}
val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester())
val testMethodName = try {
Expand All @@ -504,17 +493,11 @@ class UtBotSymbolicEngine(
}

emit(
UtExecution(
UtFuzzedExecution(
stateBefore = initialEnvironmentModels,
stateAfter = concreteExecutionResult.stateAfter,
result = concreteExecutionResult.result,
instrumentation = emptyList(),
path = mutableListOf(),
fullPath = emptyList(),
coverage = concreteExecutionResult.coverage,
createdBy = UtExecutionCreator.FUZZER,
testMethodName = testMethodName?.testName,
displayName = testMethodName?.takeIf { hasMethodUnderTestParametersToFuzz }?.displayName
coverage = concreteExecutionResult.coverage
)
)
}
Expand All @@ -524,13 +507,14 @@ class UtBotSymbolicEngine(
stateBefore: EnvironmentModels,
e: ConcreteExecutionFailureException
) {
val failedConcreteExecution = UtSymbolicExecution(
val failedConcreteExecution = UtExecution(
stateBefore = stateBefore,
stateAfter = MissingState,
result = UtConcreteExecutionFailure(e),
instrumentation = emptyList(),
path = mutableListOf(),
fullPath = listOf()
fullPath = listOf(),
createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
)

emit(failedConcreteExecution)
Expand Down Expand Up @@ -564,13 +548,14 @@ class UtBotSymbolicEngine(
val stateAfter = modelsAfter.constructStateForMethod(methodUnderTest)
require(stateBefore.parameters.size == stateAfter.parameters.size)

val symbolicUtExecution = UtSymbolicExecution(
val symbolicUtExecution = UtExecution(
stateBefore = stateBefore,
stateAfter = stateAfter,
result = symbolicExecutionResult,
instrumentation = instrumentation,
path = entryMethodPath(state),
fullPath = state.fullPath()
fullPath = state.fullPath(),
createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
)

globalGraph.traversed(state)
Expand Down
18 changes: 14 additions & 4 deletions ...mework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/context/CgContext.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,16 @@ import org.utbot.framework.codegen.model.tree.CgThisInstance
import org.utbot.framework.codegen.model.tree.CgValue
import org.utbot.framework.codegen.model.tree.CgVariable
import org.utbot.framework.codegen.model.util.createTestClassName
import org.utbot.framework.plugin.api.BuiltinClassId
import org.utbot.framework.plugin.api.ClassId
import org.utbot.framework.plugin.api.CodegenLanguage
import org.utbot.framework.plugin.api.ExecutableId
import org.utbot.framework.plugin.api.FieldId
import org.utbot.framework.plugin.api.MethodId
import org.utbot.framework.plugin.api.MockFramework
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtModel
import org.utbot.framework.plugin.api.UtReferenceModel
import java.util.IdentityHashMap
import kotlinx.collections.immutable.PersistentList
import kotlinx.collections.immutable.PersistentMap
Expand Down Expand Up @@ -166,7 +176,7 @@ internal interface CgContextOwner {
// map from a set of tests for a method to another map
// which connects code generation error message
// with the number of times it occurred
val codeGenerationErrors: MutableMap<CgMethodTestSet, MutableMap<String, Int>>
val codeGenerationErrors: MutableMap<UtMethodTestSet, MutableMap<String, Int>>

// package for generated test class
val testClassPackageName: String
Expand Down Expand Up @@ -223,8 +233,8 @@ internal interface CgContextOwner {
currentBlock = currentBlock.add(it)
}

fun updateCurrentExecutable(executableId: ExecutableId) {
currentExecutable = executableId
fun updateCurrentExecutable(method: UtMethod<*>) {
currentExecutable = method.callable.executableId
}

fun addExceptionIfNeeded(exception: ClassId) {
Expand Down Expand Up @@ -389,7 +399,7 @@ internal data class CgContext(
override val testMethods: MutableList<CgTestMethod> = mutableListOf(),
override val existingMethodNames: MutableSet<String> = mutableSetOf(),
override val prevStaticFieldValues: MutableMap<FieldId, CgVariable> = mutableMapOf(),
override val paramNames: Map<UtMethod<*>, List<String>>,
override val paramNames: Map<ExecutableId, List<String>>,
override var currentExecution: UtSymbolicExecution? = null,
override val testFramework: TestFramework,
override val mockFramework: MockFramework,
Expand Down
45 changes: 38 additions & 7 deletions ...src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,38 @@ import org.utbot.framework.codegen.model.util.resolve
import org.utbot.framework.codegen.model.util.stringLiteral
import org.utbot.framework.fields.ExecutionStateAnalyzer
import org.utbot.framework.fields.FieldPath
import org.utbot.framework.plugin.api.*
import org.utbot.framework.plugin.api.BuiltinClassId
import org.utbot.framework.plugin.api.BuiltinMethodId
import org.utbot.framework.plugin.api.ClassId
import org.utbot.framework.plugin.api.CodegenLanguage
import org.utbot.framework.plugin.api.ConcreteExecutionFailureException
import org.utbot.framework.plugin.api.ConstructorId
import org.utbot.framework.plugin.api.ExecutableId
import org.utbot.framework.plugin.api.FieldId
import org.utbot.framework.plugin.api.MethodId
import org.utbot.framework.plugin.api.TimeoutException
import org.utbot.framework.plugin.api.TypeParameters
import org.utbot.framework.plugin.api.UtArrayModel
import org.utbot.framework.plugin.api.UtAssembleModel
import org.utbot.framework.plugin.api.UtClassRefModel
import org.utbot.framework.plugin.api.UtCompositeModel
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
import org.utbot.framework.plugin.api.UtDirectSetFieldModel
import org.utbot.framework.plugin.api.UtEnumConstantModel
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionFailure
import org.utbot.framework.plugin.api.UtExecutionSuccess
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
import org.utbot.framework.plugin.api.UtModel
import org.utbot.framework.plugin.api.UtNewInstanceInstrumentation
import org.utbot.framework.plugin.api.UtNullModel
import org.utbot.framework.plugin.api.UtPrimitiveModel
import org.utbot.framework.plugin.api.UtReferenceModel
import org.utbot.framework.plugin.api.UtStaticMethodInstrumentation
import org.utbot.framework.plugin.api.UtTimeoutException
import org.utbot.framework.plugin.api.UtVoidModel
import org.utbot.framework.plugin.api.onFailure
import org.utbot.framework.plugin.api.onSuccess
import org.utbot.framework.plugin.api.util.booleanClassId
import org.utbot.framework.plugin.api.util.builtinStaticMethodId
import org.utbot.framework.plugin.api.util.doubleArrayClassId
Expand Down Expand Up @@ -278,7 +309,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
}
}

private fun processExecutionFailure(execution: UtSymbolicExecution, exception: Throwable) {
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
val methodInvocationBlock = {
with(currentExecutable) {
when (this) {
Expand Down Expand Up @@ -321,7 +352,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
methodInvocationBlock()
}

private fun shouldTestPassWithException(execution: UtSymbolicExecution, exception: Throwable): Boolean {
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
// tests with timeout or crash should be processed differently
if (exception is TimeoutException || exception is ConcreteExecutionFailureException) return false

Expand All @@ -330,7 +361,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
return exceptionRequiresAssert || exceptionIsExplicit
}

private fun shouldTestPassWithTimeoutException(execution: UtSymbolicExecution, exception: Throwable): Boolean {
private fun shouldTestPassWithTimeoutException(execution: UtExecution, exception: Throwable): Boolean {
return execution.result is UtTimeoutException || exception is TimeoutException
}

Expand Down Expand Up @@ -1061,7 +1092,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
}
}

fun createTestMethod(utMethod: UtMethod<*>, execution: UtSymbolicExecution): CgTestMethod =
fun createTestMethod(executableId: ExecutableId, execution: UtSymbolicExecution): CgTestMethod =
withTestMethodScope(execution) {
val testMethodName = nameGenerator.testMethodNameFor(executableId, execution.testMethodName)
// TODO: remove this line when SAT-1273 is completed
Expand Down Expand Up @@ -1193,7 +1224,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
}

private fun createParameterDeclarations(
testSet: UtMethodTestSet,
testSet: CgMethodTestSet,
genericExecution: UtSymbolicExecution,
Comment thread
amandelpie marked this conversation as resolved.
Outdated
): List<CgParameterDeclaration> {
val executableUnderTest = testSet.executableId
Expand Down Expand Up @@ -1307,7 +1338,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
}
}

private fun createExecutionArguments(testSet: UtMethodTestSet, execution: UtSymbolicExecution): List<CgExpression> {
private fun createExecutionArguments(testSet: CgMethodTestSet, execution: UtSymbolicExecution): List<CgExpression> {
val arguments = mutableListOf<CgExpression>()
execution.stateBefore.thisInstance?.let {
arguments += variableConstructor.getOrCreateVariable(it)
Expand Down
15 changes: 10 additions & 5 deletions utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ open class TestCaseGenerator(
val currentUtContext = utContext

val method2controller = methods.associateWith { EngineController() }
val method2executions = methods.associateWith { mutableListOf<UtSymbolicExecution>() }
val method2executions = methods.associateWith { mutableListOf<UtExecution>() }
val method2errors = methods.associateWith { mutableMapOf<String, Int>() }

runIgnoringCancellationException {
Expand All @@ -160,10 +160,15 @@ open class TestCaseGenerator(

engineActions.map { engine.apply(it) }

generate(engine).collect {
when (it) {
is UtSymbolicExecution -> method2executions.getValue(method) += it
is UtError -> method2errors.getValue(method).merge(it.description, 1, Int::plus)
generate(engine)
.catch {
logger.error(it) { "Error in flow" }
}
.collect {
when (it) {
is UtExecution -> method2executions.getValue(method) += it
is UtError -> method2errors.getValue(method).merge(it.description, 1, Int::plus)
}
}
}
controller.paused = true
Expand Down
15 changes: 14 additions & 1 deletion utbot-framework/src/test/kotlin/org/utbot/examples/UtModelTestCaseChecker.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,21 @@ import org.utbot.common.WorkaroundReason.HACK
import org.utbot.common.workaround
import org.utbot.engine.prettify
import org.utbot.framework.UtSettings.checkSolverTimeoutMillis
import org.utbot.framework.plugin.api.ClassId
import org.utbot.framework.plugin.api.CodegenLanguage
import org.utbot.framework.plugin.api.FieldId
import org.utbot.framework.plugin.api.MockStrategyApi
import org.utbot.framework.plugin.api.MockStrategyApi.NO_MOCKS
import org.utbot.framework.plugin.api.UtAssembleModel
import org.utbot.framework.plugin.api.UtCompositeModel
import org.utbot.framework.plugin.api.UtDirectSetFieldModel
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionResult
import org.utbot.framework.plugin.api.UtMethod
import org.utbot.framework.plugin.api.UtModel
import org.utbot.framework.plugin.api.UtMethodTestSet
import org.utbot.framework.plugin.api.exceptionOrNull
import org.utbot.framework.plugin.api.getOrThrow
import org.utbot.framework.plugin.api.util.UtContext
import org.utbot.framework.plugin.api.util.defaultValueModel
import org.utbot.framework.plugin.api.util.executableId
Expand All @@ -23,7 +37,6 @@ import kotlin.reflect.KFunction3
import org.junit.jupiter.api.Assertions.assertTrue
import org.utbot.framework.UtSettings.useFuzzing
import org.utbot.framework.codegen.TestCodeGeneratorPipeline
import org.utbot.framework.plugin.api.*
import org.utbot.framework.util.Conflict

internal abstract class UtModelTestCaseChecker(
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.