Implement minimalistic jc to ut execution conversion, enable codegen

UnitTestBot · EgorkaKulikov · Nov 9, 2023 · Oct 28, 2023 · Oct 30, 2023 · Oct 31, 2023
commit d79ea8bf75cd343e3db1969d937b8c696df3255c
diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle
@@ -140,6 +140,8 @@ dependencies {
     implementation group: 'org.mockito', name: 'mockito-inline', version: mockitoInlineVersion
     implementation 'junit:junit:4.13.2'
 
+    implementation "org.burningwave:core:12.62.7"
+
     implementation('org.usvm:usvm-core')
     implementation('org.usvm:usvm-jvm')
     implementation('org.usvm:usvm-jvm-instrumentation')

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -354,6 +354,9 @@ interface Tool {
 }
 
 fun main(args: Array<String>) {
+    // See https://dzone.com/articles/how-to-export-all-modules-to-all-modules-at-runtime-in-java?preview=true
+    org.burningwave.core.assembler.StaticComponentContainer.Modules.exportAllToAll()
+
     val estimatorArgs: Array<String>
     val methodFilter: String?
     val projectFilter: List<String>?

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt
@@ -11,6 +11,9 @@ import org.objectweb.asm.Type
 import org.usvm.UMachineOptions
 import org.usvm.instrumentation.util.toJcdbSignature
 import org.usvm.machine.JcMachine
+import org.usvm.machine.state.JcState
+import org.utbot.common.ThreadBasedExecutor
+import org.utbot.common.debug
 import org.utbot.common.info
 import org.utbot.common.measureTime
 import org.utbot.contest.*
@@ -24,12 +27,14 @@ import org.utbot.framework.codegen.domain.junitByVersion
 import org.utbot.framework.codegen.generator.CodeGenerator
 import org.utbot.framework.codegen.generator.CodeGeneratorParams
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
+import org.utbot.framework.minimization.minimizeExecutions
 import org.utbot.framework.plugin.api.*
 import org.utbot.framework.plugin.api.util.constructor
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.framework.plugin.api.util.method
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.fuzzer.UtFuzzedExecution
+import org.utbot.summary.summarizeAll
 import java.io.File
 import java.net.URLClassLoader
 import kotlin.math.max
@@ -98,6 +103,7 @@ fun runUsvmGeneration(
 
     //remaining budget
     val startTime = System.currentTimeMillis()
+    logger.debug { "STARTED COUNTING BUDGET FOR ${cut.classId.name}" }
     fun remainingBudgetMillisWithoutCodegen() =
         max(0, generationTimeoutMillisWithoutCodegen - (System.currentTimeMillis() - startTime))
 
@@ -139,14 +145,16 @@ fun runUsvmGeneration(
         val totalBudgetPerMethod = remainingBudgetMillisWithoutCodegen() / filteredMethods.size
         val concreteBudgetMsPerMethod = 500L
             .coerceIn((totalBudgetPerMethod / 10L).. (totalBudgetPerMethod / 2L))
+        val symbolicBudgetPerMethod = totalBudgetPerMethod - concreteBudgetMsPerMethod
+        logger.debug { "Symbolic budget per method: $symbolicBudgetPerMethod" }
 
         // TODO usvm-sbft: reuse same machine for different classes,
         //  right now I can't do that, because `timeoutMs` can't be changed after machine creation
         logger.info().measureTime({ "preparation: creating JcMachine" }) {
             JcMachine(
                 cp = jcDbContainer.cp,
                 // TODO usvm-sbft: we may want to tune UMachineOptions for contest
-                options = UMachineOptions(timeoutMs = totalBudgetPerMethod - concreteBudgetMsPerMethod)
+                options = UMachineOptions(timeoutMs = symbolicBudgetPerMethod)
             )
         }.use { machine ->
             val jcClass = jcDbContainer.cp.findClass(cut.fqn)
@@ -167,7 +175,17 @@ fun runUsvmGeneration(
                     logger.error { "Method [$method] not found in jcClass [$jcClass]" }
                     continue
                 }
-                val states = machine.analyze(jcTypedMethod.method)
+                val states = logger.debug().measureTime({ "machine.analyze(${method.classId}.${method.signature})" }) {
+                    ((ThreadBasedExecutor.threadLocal.invokeWithTimeout(10 * symbolicBudgetPerMethod) {
+                        machine.analyze(jcTypedMethod.method)
+                    } as? Result<List<JcState>>) ?: run {
+                        logger.error { "machine.analyze(${jcTypedMethod.method}) timed out" }
+                        Result.success(emptyList())
+                    }).getOrElse { e ->
+                        logger.error("JcMachine failed", e)
+                        emptyList()
+                    }
+                }
                 val jcExecutions = states.mapNotNull {
                     // TODO usvm-sbft: if we have less than `runner.timeout` budget we should only let resolver run
                     //  for `remainingBudgetMillisWithoutCodegen()` ms, right now last resolver call may exceed budget,
@@ -176,11 +194,13 @@ fun runUsvmGeneration(
                         // TODO usvm-sbft: right now this call fails unless you set:
                         //  - "usvm-jvm-instrumentation-jar" environment variable to something like "/home/ilya/IdeaProjects/usvm/usvm-jvm-instrumentation/build/libs/usvm-jvm-instrumentation-1.0.jar"
                         //  - "usvm-jvm-collectors-jar"  environment variable to something like "/home/ilya/IdeaProjects/usvm/usvm-jvm-instrumentation/build/libs/usvm-jvm-instrumentation-collectors.jar"
-                        runCatching {
-                            resolver.resolve(jcTypedMethod, it)
-                        }.getOrElse { e ->
-                            logger.error(e) { "Resolver failed" }
-                            null
+                        logger.debug().measureTime({ "resolver.resolve(${method.classId}.${method.signature}, ...)" }) {
+                            runCatching {
+                                resolver.resolve(jcTypedMethod, it)
+                            }.getOrElse { e ->
+                                logger.error(e) { "Resolver failed" }
+                                null
+                            }
                         }
                     else null
                 }
@@ -193,7 +213,9 @@ fun runUsvmGeneration(
                 statsForClass.statsForMethods.add(statsForMethod)
 
                 val utExecutions: List<UtExecution> = jcExecutions.mapNotNull {
-                    JcToUtExecutionConverter(instructionIdProvider).convert(it)
+                    logger.debug().measureTime({ "Convert JcExecution" }) {
+                        JcToUtExecutionConverter(instructionIdProvider).convert(it)
+                    }
                 }
 
                 utExecutions.forEach { result ->
@@ -220,19 +242,20 @@ fun runUsvmGeneration(
                         logger.error(e) { "Test generation failed during stats update" }
                     }
                 }
+                logger.debug { "Finished $method" }
             }
         }
     }
 
-// TODO usvm-sbft: codegen, requires proper UtUsvmExecution creation (not just coverage)
+    val testSets = testsByMethod.map { (method, executions) ->
+        UtMethodTestSet(method, minimizeExecutions(executions), jimpleBody = null)
+    }.summarizeAll(cut.classfileDir.toPath(), sourceFile = null)
+
+    logger.info().measureTime({ "Flushing tests for [${cut.simpleName}] on disk" }) {
+        writeTestClass(cut, codeGenerator.generateAsString(testSets))
+    }
 
-//    val testSets = testsByMethod.map { (method, executions) ->
-//        UtMethodTestSet(method, minimizeExecutions(executions), jimpleBody(method))
-//    }.summarizeAll(cut.classfileDir.toPath(), sourceFile = null)
-//
-//    logger.info().measureTime({ "Flushing tests for [${cut.simpleName}] on disk" }) {
-//        writeTestClass(cut, codeGenerator.generateAsString(testSets))
-//    }
+    logger.debug { "STOPPED COUNTING BUDGET FOR ${cut.classId.name}" }
 
     statsForClass
 }
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ConverterUtils.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ConverterUtils.kt
@@ -2,17 +2,21 @@ package org.utbot.contest.usvm
 
 import org.jacodb.analysis.library.analyzers.thisInstance
 import org.jacodb.api.JcClassOrInterface
+import org.jacodb.api.JcField
 import org.jacodb.api.JcMethod
 import org.jacodb.api.JcType
 import org.jacodb.api.TypeName
 import org.usvm.instrumentation.testcase.api.UTestInst
 import org.usvm.instrumentation.testcase.descriptor.UTestObjectDescriptor
 import org.usvm.instrumentation.testcase.descriptor.UTestValueDescriptor
 import org.usvm.instrumentation.util.toJavaClass
+import org.usvm.instrumentation.util.toJavaField
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConstructorId
 import org.utbot.framework.plugin.api.ExecutableId
+import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.MethodId
+import org.utbot.framework.plugin.api.util.fieldId
 import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.objectClassId
 import org.utbot.framework.plugin.api.util.utContext
@@ -37,5 +41,8 @@ val JcClassOrInterface.classId: ClassId
 val TypeName.classId: ClassId
     get() = ClassId(this.typeName)
 
+val JcField.fieldId: FieldId
+    get() = toJavaField(utContext.classLoader)!!.fieldId
+
 val UTestValueDescriptor.origin: UTestInst?
     get() = (this as UTestObjectDescriptor)?.originUTestExpr
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/JcToUtExecutionConverter.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/JcToUtExecutionConverter.kt
@@ -1,9 +1,10 @@
 package org.utbot.contest.usvm
 
+import mu.KotlinLogging
 import org.jacodb.api.JcClassOrInterface
-import org.jacodb.api.JcMethod
 import org.jacodb.api.JcTypedMethod
 import org.jacodb.api.cfg.JcInst
+import org.jacodb.api.cfg.JcThrowInst
 import org.jacodb.api.ext.jcdbSignature
 import org.usvm.instrumentation.testcase.api.UTestExecutionExceptionResult
 import org.usvm.instrumentation.testcase.api.UTestExecutionFailedResult
@@ -13,30 +14,26 @@ import org.usvm.instrumentation.testcase.api.UTestExecutionState
 import org.usvm.instrumentation.testcase.api.UTestExecutionSuccessResult
 import org.usvm.instrumentation.testcase.api.UTestExecutionTimedOutResult
 import org.usvm.instrumentation.testcase.descriptor.Descriptor2ValueConverter
+import org.usvm.instrumentation.testcase.descriptor.UTestExceptionDescriptor
 import org.usvm.instrumentation.util.enclosingClass
 import org.usvm.instrumentation.util.enclosingMethod
-import org.usvm.instrumentation.util.toJavaField
 import org.utbot.contest.usvm.executor.JcExecution
 import org.utbot.framework.codegen.domain.builtin.TestClassUtilMethodProvider
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.Coverage
 import org.utbot.framework.plugin.api.EnvironmentModels
 import org.utbot.framework.plugin.api.ExecutableId
-import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.Instruction
-import org.utbot.framework.plugin.api.MethodId
-import org.utbot.framework.plugin.api.MissingState
 import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.framework.plugin.api.UtExecutionFailure
 import org.utbot.framework.plugin.api.UtExecutionSuccess
-import org.utbot.framework.plugin.api.UtVoidModel
-import org.utbot.framework.plugin.api.util.fieldId
-import org.utbot.framework.plugin.api.util.jClass
+import org.utbot.framework.plugin.api.UtExplicitlyThrownException
+import org.utbot.framework.plugin.api.UtImplicitlyThrownException
+import org.utbot.framework.plugin.api.UtInstrumentation
 import org.utbot.framework.plugin.api.util.objectClassId
 import org.utbot.framework.plugin.api.util.utContext
-import org.utbot.instrumentation.instrumentation.execution.constructors.StateBeforeAwareIdGenerator
-import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
-import org.utbot.instrumentation.instrumentation.execution.constructors.javaStdLibModelWithCompositeOriginConstructors
-import java.util.*
+
+private val logger = KotlinLogging.logger {}
 
 class JcToUtExecutionConverter(
     private val instructionIdProvider: InstructionIdProvider,
@@ -45,50 +42,65 @@ class JcToUtExecutionConverter(
     private val testClassId: ClassId = objectClassId
     private val utilMethodProvider = TestClassUtilMethodProvider(testClassId)
 
-    private val modelConverter = JcToUtModelConverter(utilMethodProvider)
+    private val modelConverter = JcToUtModelConverter()
     private val instConverter = UTestInst2UtModelConverter(utilMethodProvider)
+    private val toValueConverter = Descriptor2ValueConverter(utContext.classLoader)
 
     fun convert(jcExecution: JcExecution): UtExecution? {
         val coverage = convertCoverage(getTrace(jcExecution.uTestExecutionResult), jcExecution.method.enclosingType.jcClass)
+        val instrumentation = emptyList<UtInstrumentation>() // TODO usvm-sbft: fill up instrumentation with data from UTest
 
-        val executionResult = jcExecution.uTestExecutionResult
-        return when (executionResult) {
-            is UTestExecutionSuccessResult -> {
-                val result = UtExecutionSuccess(modelConverter.convert(executionResult.result))
-
-
+        return when (val executionResult = jcExecution.uTestExecutionResult) {
+            is UTestExecutionSuccessResult -> UtUsvmExecution(
+                stateBefore = convertState(executionResult.initialState, jcExecution.method),
+                stateAfter = convertState(executionResult.resultState, jcExecution.method),
+                result = UtExecutionSuccess(modelConverter.convert(executionResult.result)),
+                coverage = coverage,
+                instrumentation = instrumentation,
+            )
+            is UTestExecutionExceptionResult -> toUserRaisedException(executionResult.cause)?.let { exception ->
                 UtUsvmExecution(
-                    stateBefore = MissingState,
-                    stateAfter = MissingState,
-                    result = result,
+                    stateBefore = convertState(executionResult.initialState, jcExecution.method),
+                    stateAfter = convertState(executionResult.resultState, jcExecution.method),
+                    result = createExecutionFailureResult(
+                        exception,
+                        jcExecution.method,
+                        executionResult.trace
+                    ),
                     coverage = coverage,
-                    instrumentation = emptyList()
+                    instrumentation = instrumentation,
                 )
             }
-
-            is UTestExecutionExceptionResult -> TODO("usvm-sbft")
             is UTestExecutionInitFailedResult -> {
-                val exception =
-                    valueConstructor.buildObjectFromDescriptor(jcExecution.uTestExecutionResult.cause) as Throwable
-                logger.error(exception) { "Concrete executor failed" }
+                toUserRaisedException(executionResult.cause)?.let { e ->
+                    logger.warn(e) { "Execution failed before method under test call" }
+                }
                 null
             }
 
             is UTestExecutionFailedResult -> {
-                val exception =
-                    valueConstructor.buildObjectFromDescriptor(jcExecution.uTestExecutionResult.cause) as Throwable
-                if (!jcExecution.uTestExecutionResult.cause.raisedByUserCode)
-                    logger.error(exception) { "Concrete executor failed" }
-                // TODO usvm-sbft
-                null
+                toUserRaisedException(executionResult.cause)?.let { e ->
+                    // TODO usvm-sbft
+                    null
+                }
             }
 
             is UTestExecutionTimedOutResult -> {
                 // TODO usvm-sbft
                 null
             }
         }
+        // TODO usvm-sbft: deep map UtExecution to substitute models build from UTest
+    }
 
+    private fun toUserRaisedException(exceptionDescriptor: UTestExceptionDescriptor): Throwable? {
+        val exception =
+            toValueConverter.buildObjectFromDescriptor(exceptionDescriptor) as Throwable
+        return if (exceptionDescriptor.raisedByUserCode) exception
+        else {
+            logger.error(exception) { "Concrete execution failed" }
+            null
+        }
     }
 
     private fun getTrace(executionResult: UTestExecutionResult): List<JcInst>? = when (executionResult) {
@@ -100,21 +112,34 @@ class JcToUtExecutionConverter(
     }
 
     private fun convertState(state: UTestExecutionState, method: JcTypedMethod): EnvironmentModels {
-        val thisInstance = instConverter.convert(state.instanceDescriptor?.origin!!)
-        val parameters = state.argsDescriptors.map { instConverter.convert(it?.origin!!) }
+        val thisInstance =
+            if (method.isStatic) null
+            else modelConverter.convert(state.instanceDescriptor)
+        val parameters = state.argsDescriptors.map { modelConverter.convert(it) }
         val statics = state.statics
             .entries
             .associate { (jcField, uTestDescr) ->
-                val fieldType = jcField.toJavaField(utContext.classLoader)!!.fieldId.type
-                val fieldId = FieldId(fieldType, jcField.name)
-                val fieldModel = modelConverter.convert(uTestDescr)
-
-                fieldId to fieldModel
+                jcField.fieldId to modelConverter.convert(uTestDescr)
             }
         val executableId: ExecutableId = method.method.toExecutableId()
         return EnvironmentModels(thisInstance, parameters, statics, executableId)
     }
 
+    private fun createExecutionFailureResult(
+        exception: Throwable,
+        jcTypedMethod: JcTypedMethod,
+        jcCoverage: List<JcInst>?
+    ): UtExecutionFailure {
+        // TODO usvm-sbft: test that exceptions are correctly classified
+        val lastJcInst = jcCoverage.orEmpty().lastOrNull()
+            ?: return UtImplicitlyThrownException(exception, fromNestedMethod = false)
+        val fromNestedMethod = lastJcInst.enclosingMethod != jcTypedMethod.method
+        return if (lastJcInst is JcThrowInst)
+            UtExplicitlyThrownException(exception, fromNestedMethod)
+        else
+            UtImplicitlyThrownException(exception, fromNestedMethod)
+    }
+
     private fun convertCoverage(jcCoverage: List<JcInst>?, jcClass: JcClassOrInterface) = Coverage(
         coveredInstructions = jcCoverage.orEmpty().map {
             val methodSignature = it.enclosingMethod.jcdbSignature