Make TestCaseGenerator use UsvmSymbolicEngine for entire class (#…

…2716)
UnitTestBot · EgorkaKulikov · Dec 11, 2023 · Dec 11, 2023 · Dec 11, 2023 · Dec 13, 2023
commit 5d688394938916fc0742347c41f506fff5a7ca1a
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UsvmSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UsvmSymbolicEngine.kt
@@ -2,9 +2,15 @@ package org.utbot.engine
 
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.emptyFlow
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.UtResult
 
-class UsvmSymbolicEngine {
-
-    fun generateWithUsvm(): Flow<UtResult> = emptyFlow()
+object UsvmSymbolicEngine {
+    // TODO implement
+    fun runUsvmGeneration(
+        methods: List<ExecutableId>,
+        classpath: String,
+        timeoutMillis: Long
+    ): Flow<Pair<ExecutableId, UtResult>> =
+        emptyFlow()
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt
@@ -8,6 +8,7 @@ import kotlinx.coroutines.flow.catch
 import kotlinx.coroutines.flow.emitAll
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.yield
@@ -16,6 +17,7 @@ import mu.KotlinLogging
 import org.utbot.common.*
 import org.utbot.engine.EngineController
 import org.utbot.engine.Mocker
+import org.utbot.engine.UsvmSymbolicEngine
 import org.utbot.engine.UtBotSymbolicEngine
 import org.utbot.engine.util.mockListeners.ForceMockListener
 import org.utbot.engine.util.mockListeners.ForceStaticMockListener
@@ -173,9 +175,10 @@ open class TestCaseGenerator(
         methods: List<ExecutableId>,
         mockStrategy: MockStrategyApi,
         chosenClassesToMockAlways: Set<ClassId> = Mocker.javaDefaultClasses.mapTo(mutableSetOf()) { it.id },
-        methodsGenerationTimeout: Long = utBotGenerationTimeoutInMillis,
+        utBotTimeout: Long = utBotGenerationTimeoutInMillis,
         userTaintConfigurationProvider: TaintConfigurationProvider? = null,
-        generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = defaultTestFlow(methodsGenerationTimeout)
+        generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = defaultTestFlow(utBotTimeout),
+        usvmTimeoutMillis: Long = 0,
     ): List<UtMethodTestSet> = ConcreteExecutor.defaultPool.use { _ -> // TODO: think on appropriate way to close instrumented processes
         if (isCanceled()) return@use methods.map { UtMethodTestSet(it) }
 
@@ -189,7 +192,7 @@ open class TestCaseGenerator(
             return@use methods.map { method -> UtMethodTestSet(method, errors = method2errors.getValue(method)) }
 
         val executionStartInMillis = System.currentTimeMillis()
-        val executionTimeEstimator = ExecutionTimeEstimator(methodsGenerationTimeout, methods.size)
+        val executionTimeEstimator = ExecutionTimeEstimator(utBotTimeout, methods.size)
 
         val currentUtContext = utContext
 
@@ -200,6 +203,29 @@ open class TestCaseGenerator(
         val forceMockListener = ForceMockListener.create(this, conflictTriggers)
         val forceStaticMockListener = ForceStaticMockListener.create(this, conflictTriggers)
 
+        suspend fun consumeUtResultFlow(utResultFlow: Flow<Pair<ExecutableId, UtResult>>) =
+            utResultFlow.catch {
+                logger.error(it) { "Error in flow" }
+            }
+                .collect { (executableId, utResult) ->
+                    when (utResult) {
+                        is UtExecution -> {
+                            if (utResult is UtSymbolicExecution &&
+                                (conflictTriggers.triggered(Conflict.ForceMockHappened) ||
+                                        conflictTriggers.triggered(Conflict.ForceStaticMockHappened))
+                            ) {
+                                utResult.containsMocking = true
+                            }
+                            method2executions.getValue(executableId) += utResult
+                        }
+
+                        is UtError -> {
+                            method2errors.getValue(executableId).merge(utResult.description, 1, Int::plus)
+                            logger.error(utResult.error) { "UtError occurred" }
+                        }
+                    }
+                }
+
         runIgnoringCancellationException {
             runBlockingWithCancellationPredicate(isCanceled) {
                 for ((method, controller) in method2controller) {
@@ -223,27 +249,7 @@ open class TestCaseGenerator(
                             engineActions.map { engine.apply(it) }
                             engineActions.clear()
 
-                            generate(engine)
-                                .catch {
-                                    logger.error(it) { "Error in flow" }
-                                }
-                                .collect {
-                                    when (it) {
-                                        is UtExecution -> {
-                                            if (it is UtSymbolicExecution &&
-                                                (conflictTriggers.triggered(Conflict.ForceMockHappened) ||
-                                                        conflictTriggers.triggered(Conflict.ForceStaticMockHappened))
-                                            ) {
-                                                it.containsMocking = true
-                                            }
-                                            method2executions.getValue(method) += it
-                                        }
-                                        is UtError -> {
-                                            method2errors.getValue(method).merge(it.description, 1, Int::plus)
-                                            logger.error(it.error) { "UtError occurred" }
-                                        }
-                                    }
-                                }
+                            consumeUtResultFlow(generate(engine).map { utResult -> method to utResult })
                         } catch (e: Exception) {
                             logger.error(e) {"Error in engine"}
                             throw e
@@ -284,6 +290,8 @@ open class TestCaseGenerator(
                     }
                     logger.debug("test generator global scope lifecycle check ended")
                 }
+
+                consumeUtResultFlow(UsvmSymbolicEngine.runUsvmGeneration(methods, classpathForEngine, usvmTimeoutMillis))
             }
         }
 

diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestFlow.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestFlow.kt
@@ -4,10 +4,8 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.emptyFlow
 import kotlinx.coroutines.flow.flattenConcat
 import kotlinx.coroutines.flow.flowOf
-import org.utbot.engine.UsvmSymbolicEngine
 import org.utbot.engine.UtBotSymbolicEngine
 import org.utbot.framework.UtSettings
-import org.utbot.framework.codegen.domain.SymbolicEngineSource
 
 /**
  * Constructs [TestFlow] for customization and creates flow producer.
@@ -18,8 +16,7 @@ fun testFlow(block: TestFlow.() -> Unit): UtBotSymbolicEngine.() -> Flow<UtResul
  * Creates default flow that uses [UtSettings] for customization.
  */
 fun defaultTestFlow(timeout: Long) = testFlow {
-    isSymbolicEngineEnabled = true
-    symbolicEngineType = SymbolicEngineSource.UnitTestBot
+    isUtBotSymbolicEngineEnabled = true
     generationTimeout = timeout
     isFuzzingEnabled = UtSettings.useFuzzing
     if (generationTimeout > 0) {
@@ -42,8 +39,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
         set(value) {
             field = maxOf(0, value)
         }
-    var isSymbolicEngineEnabled = true
-    var symbolicEngineType = SymbolicEngineSource.UnitTestBot
+    var isUtBotSymbolicEngineEnabled = true
     var isFuzzingEnabled = false
     var fuzzingValue: Double = 0.1
         set(value) {
@@ -64,7 +60,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
         return when {
             generationTimeout == 0L -> emptyFlow()
             isFuzzingEnabled -> {
-                when (val value = if (isSymbolicEngineEnabled) (fuzzingValue * generationTimeout).toLong() else generationTimeout) {
+                when (val value = if (isUtBotSymbolicEngineEnabled) (fuzzingValue * generationTimeout).toLong() else generationTimeout) {
                     0L -> engine.traverse()
                     generationTimeout -> engine.fuzzing(System.currentTimeMillis() + value)
                     else -> flowOf(
@@ -73,12 +69,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
                     ).flattenConcat()
                 }
             }
-            isSymbolicEngineEnabled -> {
-                when (symbolicEngineType) {
-                    SymbolicEngineSource.UnitTestBot -> engine.traverse()
-                    SymbolicEngineSource.Usvm -> UsvmSymbolicEngine().generateWithUsvm()
-                }
-            }
+            isUtBotSymbolicEngineEnabled -> engine.traverse()
             else -> emptyFlow()
         }
     }

diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt
@@ -13,6 +13,7 @@ import org.utbot.framework.codegen.domain.ParametrizedTestSource
 import org.utbot.framework.codegen.domain.ProjectType
 import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
 import org.utbot.framework.codegen.domain.SymbolicEngineSource
+import org.utbot.framework.codegen.domain.SymbolicEngineSource.*
 import org.utbot.framework.codegen.domain.testFrameworkByName
 import org.utbot.framework.codegen.generator.AbstractCodeGenerator
 import org.utbot.framework.codegen.generator.CodeGeneratorParams
@@ -114,10 +115,17 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
         val methods: List<ExecutableId> = kryoHelper.readObject(params.methods)
         logger.debug()
             .measureTime({ "starting generation for ${methods.size} methods, starting with ${methods.first()}" }) {
+                val symbolicEngineSource = SymbolicEngineSource.valueOf(params.symbolicEngineType)
+                val utBotTimeout = when {
+                    !params.isSymbolicEngineEnabled -> params.timeout
+                    symbolicEngineSource == UnitTestBot -> params.timeout
+                    params.isFuzzingEnabled -> (params.fuzzingValue * params.timeout).toLong()
+                    else -> 0L
+                }
+
                 val generateFlow = testFlow {
-                    generationTimeout = params.generationTimeout
-                    isSymbolicEngineEnabled = params.isSymbolicEngineEnabled
-                    symbolicEngineType = SymbolicEngineSource.valueOf(params.symbolicEngineType)
+                    generationTimeout = utBotTimeout
+                    isUtBotSymbolicEngineEnabled = params.isSymbolicEngineEnabled && symbolicEngineSource == UnitTestBot
                     isFuzzingEnabled = params.isFuzzingEnabled
                     fuzzingValue = params.fuzzingValue
                 }
@@ -130,9 +138,10 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
                     methods,
                     MockStrategyApi.valueOf(params.mockStrategy),
                     kryoHelper.readObject(params.chosenClassesToMockAlways),
-                    params.timeout,
+                    utBotTimeout,
                     userTaintConfigurationProvider,
                     generate = generateFlow,
+                    usvmTimeoutMillis = params.generationTimeout - utBotTimeout,
                 )
                     .summarizeAll(Paths.get(params.searchDirectory), null)
                     .filterNot { it.executions.isEmpty() && it.errors.isEmpty() }