Skip to content

Add sarif results minimization #490 #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mmvpm merged 5 commits into from
Jul 13, 2022
Merged

Add sarif results minimization #490 #498

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8e396f0
Add sarif results minimization (#490)
mmvpm Jul 11, 2022
d8f34a6
Add unit tests
mmvpm Jul 12, 2022
1a08be9
Fix typos in comments
mmvpm Jul 12, 2022
587cf11
Fix after review
mmvpm Jul 12, 2022
c3c2b03
Fix after review
mmvpm Jul 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Fix after review
  • Loading branch information
@mmvpm
mmvpm committed Jul 13, 2022
commit c3c2b03674609b09a413f664bbdd5b34694e710a
17 changes: 17 additions & 0 deletions utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,25 @@ class SarifReport(

/**
* Minimizes detected errors and removes duplicates.
*
* Between two [SarifResult]s with the same `ruleId` and `locations`
* it chooses the one with the shorter length of the execution trace.
*
* __Example:__
*
* The SARIF report for the code below contains only one unchecked exception in `methodB`.
* But without minimization, the report will contain two results: for `methodA` and for `methodB`.
*
* ```
* class Example {
* int methodA(int a) {
* return methodB(a);
* }
* int methodB(int b) {
* return 1 / b;
* }
* }
* ```
*/
private fun minimizeResults(sarifResults: List<SarifResult>): List<SarifResult> {
val groupedResults = sarifResults.groupBy { sarifResult ->
Expand Down
61 changes: 59 additions & 2 deletions utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,12 +201,66 @@ class SarifReportTest {
val report = SarifReport(
testCases = testCases,
generatedTestsCode = "",
sourceFindingEmpty
sourceFindingMain
).createReport().toSarif()

assert(report.runs.first().results.size == 1) // no duplicates
}

@Test
fun testMinimizationDoesNotRemoveResultsWithDifferentRuleId() {
mockUtMethodNames()

val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)
val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)

// different ruleId's
Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false))
Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(ArithmeticException(), false))

val testCases = listOf(
UtTestCase(mockUtMethod, listOf(mockUtExecution1)),
UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // not a duplicate
)

val report = SarifReport(
testCases = testCases,
generatedTestsCode = "",
sourceFindingMain
).createReport().toSarif()

assert(report.runs.first().results.size == 2) // no results have been removed
}

@Test
fun testMinimizationDoesNotRemoveResultsWithDifferentLocations() {
mockUtMethodNames()

val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)
val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)

// the same ruleId's
Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false))
Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false))

// different locations
Mockito.`when`(mockUtExecution1.path.lastOrNull()?.stmt?.javaSourceStartLineNumber).thenReturn(11)
Mockito.`when`(mockUtExecution2.path.lastOrNull()?.stmt?.javaSourceStartLineNumber).thenReturn(22)

val testCases = listOf(
UtTestCase(mockUtMethod, listOf(mockUtExecution1)),
UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // not a duplicate
)

val report = SarifReport(
testCases = testCases,
generatedTestsCode = "",
sourceFindingMain
).createReport().toSarif()

assert(report.runs.first().results.size == 2) // no results have been removed
}

@Test
fun testMinimizationChoosesShortestCodeFlow() {
mockUtMethodNames()
Expand All @@ -216,9 +270,12 @@ class SarifReportTest {

val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)
val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS)

// the same ruleId's
Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(mockNPE1, false))
Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(mockNPE2, false))

// but different stack traces
val stackTraceElement1 = StackTraceElement("Main", "main", "Main.java", 3)
val stackTraceElement2 = StackTraceElement("Main", "main", "Main.java", 7)
Mockito.`when`(mockNPE1.stackTrace).thenReturn(arrayOf(stackTraceElement1))
Expand All @@ -232,7 +289,7 @@ class SarifReportTest {
val report = SarifReport(
testCases = testCases,
generatedTestsCode = "",
sourceFindingEmpty
sourceFindingMain
).createReport().toSarif()

assert(report.runs.first().results.size == 1) // no duplicates
Expand Down