Limit number of Spring context resets during reruns (UnitTestBot#2626)

IlyaMuravjov · web-flow · commit b42b698a074a · 2023-10-02T12:52:51.000+03:00
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt
@@ -597,6 +597,21 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
     var disableUnsatChecking by getBooleanProperty(false)
 
     // endregion
+
+    // region Spring-related options
+
+    /**
+     * When generating integration tests we only partially reset context in between executions to save time.
+     * For example, entity id generators do not get reset. It may lead to non-reproduceable results if
+     * IDs leak to the output of the method under test.
+     *
+     * To cope with that, we rerun executions that are left after minimization, fully resetting Spring context
+     * between executions. However, full context reset is slow, so we use this setting to limit number of
+     * tests per method that are rerun with full context reset in case minimization outputs too many tests.
+     */
+    var maxSpringContextResetsPerMethod by getIntProperty(25, 0, Int.MAX_VALUE)
+
+    // endregion
 }
 
 /**
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt
@@ -13,6 +13,7 @@ import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrument
 
 class RerunningConcreteExecutionContext(
     private val delegateContext: ConcreteExecutionContext,
+    private val maxRerunsPerMethod: Int,
     private val rerunTimeoutInMillis: Long = 10L * UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis,
 ) : ConcreteExecutionContext by delegateContext {
     companion object {
@@ -23,31 +24,40 @@ class RerunningConcreteExecutionContext(
         executions: List<UtExecution>,
         methodUnderTest: ExecutableId,
         rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
-    ): List<UtExecution> = delegateContext.transformExecutionsAfterMinimization(
-        executions,
-        methodUnderTest,
-        rerunExecutor
-    ).map { execution ->
-        runBlocking {
-            val result = try {
-                rerunExecutor.executeConcretely(
-                    methodUnderTest = methodUnderTest,
-                    stateBefore = execution.stateBefore,
-                    instrumentation = emptyList(),
-                    timeoutInMillis = rerunTimeoutInMillis,
-                    isRerun = true,
-                )
-            } catch (e: Throwable) {
-                // we can't update execution result if we don't have a result
-                logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" }
-                return@runBlocking execution
-            }
-            execution.copy(
-                stateBefore = result.stateBefore,
-                stateAfter = result.stateAfter,
-                result = result.result,
-                coverage = result.coverage,
-            )
-        }
+    ): List<UtExecution> {
+        @Suppress("NAME_SHADOWING")
+        val executions = delegateContext.transformExecutionsAfterMinimization(
+            executions,
+            methodUnderTest,
+            rerunExecutor
+        )
+            // it's better to rerun executions with non-empty coverage,
+            // because executions with empty coverage are often duplicated
+            .sortedBy { it.coverage?.coveredInstructions.isNullOrEmpty() }
+        return executions
+            .take(maxRerunsPerMethod)
+            .map { execution ->
+                runBlocking {
+                    val result = try {
+                        rerunExecutor.executeConcretely(
+                            methodUnderTest = methodUnderTest,
+                            stateBefore = execution.stateBefore,
+                            instrumentation = emptyList(),
+                            timeoutInMillis = rerunTimeoutInMillis,
+                            isRerun = true,
+                        )
+                    } catch (e: Throwable) {
+                        // we can't update execution result if we don't have a result
+                        logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" }
+                        return@runBlocking execution
+                    }
+                    execution.copy(
+                        stateBefore = result.stateBefore,
+                        stateAfter = result.stateAfter,
+                        result = result.result,
+                        coverage = result.coverage,
+                    )
+                }
+            } + executions.drop(maxRerunsPerMethod)
     }
 }
diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt
@@ -5,6 +5,7 @@ import org.utbot.common.dynamicPropertiesOf
 import org.utbot.common.isAbstract
 import org.utbot.common.isStatic
 import org.utbot.common.withValue
+import org.utbot.framework.UtSettings
 import org.utbot.framework.codegen.generator.AbstractCodeGenerator
 import org.utbot.framework.codegen.generator.CodeGeneratorParams
 import org.utbot.framework.codegen.generator.SpringCodeGenerator
@@ -118,7 +119,8 @@ class SpringApplicationContextImpl(
                         delegateConcreteExecutionContext,
                         classpathWithoutDependencies,
                         springApplicationContext = this
-                    )
+                    ),
+                    maxRerunsPerMethod = UtSettings.maxSpringContextResetsPerMethod
                 )
         }.transformInstrumentationFactory { delegateInstrumentationFactory ->
             RemovingConstructFailsUtExecutionInstrumentation.Factory(delegateInstrumentationFactory)
