MBS-11268 remove suppressed failed tests from verdict (#1003)

slightly changed text

was:
```
Failed. There are 1 not reported tests
Failed. There are 272 not suppressed failed tests
```

now:
```
Test suite failed. Problems:
 - Not reported tests: 1
 - Not suppressed failed tests: 272
```

LOST -> NOT REPORTED in problem tests

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/action/AvitoReportViewerFinishAction.kt

@@ -9,8 +9,8 @@ internal class AvitoReportViewerFinishAction(
 
     override fun action(verdict: Verdict) {
         if (verdict is Verdict.Failure) {
-            if (verdict.lostTests.isNotEmpty()) {
-                legacyReport.sendLostTests(verdict.lostTests)
+            if (verdict.notReportedTests.isNotEmpty()) {
+                legacyReport.sendLostTests(verdict.notReportedTests)
             }
         }
         legacyReport.finish()

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/action/SendMetricsAction.kt

@@ -9,8 +9,8 @@ internal class SendMetricsAction(
 
     override fun action(verdict: Verdict) {
         if (verdict is Verdict.Failure) {
-            if (verdict.lostTests.isNotEmpty()) {
-                metricsSender.sendNotReportedCount(verdict.lostTests.size)
+            if (verdict.notReportedTests.isNotEmpty()) {
+                metricsSender.sendNotReportedCount(verdict.notReportedTests.size)
             }
         }
     }

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/action/WriteTaskVerdictAction.kt

@@ -36,12 +36,15 @@ internal class WriteTaskVerdictAction(
                 "OK. Failed tests were suppressed"
 
             is Verdict.Failure -> buildString {
-                if (lostTests.isNotEmpty()) {
-                    appendLine("Failed. There are ${lostTests.size} not reported tests")
+                appendLine("Test suite failed. Problems:")
+                appendLine()
+
+                if (notReportedTests.isNotEmpty()) {
+                    appendLine(" - Not reported tests: ${notReportedTests.size} ")
                 }
 
-                if (failedTests.isNotEmpty()) {
-                    appendLine("Failed. There are ${failedTests.size} not suppressed failed tests")
+                if (unsuppressedFailedTests.isNotEmpty()) {
+                    appendLine(" - Not suppressed failed tests: ${unsuppressedFailedTests.size} ")
                 }
             }
         }
@@ -53,8 +56,8 @@ internal class WriteTaskVerdictAction(
                 emptySet()
 
             is Verdict.Failure ->
-                failedTests.map { test -> test.toTaskVerdictTest("FAILED") }.toSet() +
-                    lostTests.map { test -> test.toTaskVerdictTest("LOST") }.toSet()
+                unsuppressedFailedTests.map { test -> test.toTaskVerdictTest("FAILED") }.toSet() +
+                    notReportedTests.map { test -> test.toTaskVerdictTest("NOT REPORTED") }.toSet()
         }
     }
 

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/verdict/TestStatisticsCounterImpl.kt

@@ -20,10 +20,10 @@ internal class TestStatisticsCounterImpl(private val verdict: Verdict) : TestSta
     override fun skippedCount(): Int = verdict.testResults.filterIsInstance<AndroidTest.Skipped>().size
 
     override fun failureCount(): Int =
-        if (verdict is Verdict.Failure) verdict.failedTests.size else 0
+        if (verdict is Verdict.Failure) verdict.unsuppressedFailedTests.size else 0
 
     override fun notReportedCount(): Int =
-        if (verdict is Verdict.Failure) verdict.lostTests.size else 0
+        if (verdict is Verdict.Failure) verdict.notReportedTests.size else 0
 
     private fun completedTests(): List<AndroidTest.Completed> {
         return verdict.testResults.filterIsInstance<AndroidTest.Completed>()

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/verdict/Verdict.kt

@@ -19,7 +19,7 @@ internal sealed class Verdict {
 
     data class Failure(
         override val testResults: Collection<AndroidTest>,
-        val lostTests: Collection<AndroidTest.Lost>,
-        val failedTests: Collection<TestStaticData>
+        val notReportedTests: Collection<AndroidTest.Lost>,
+        val unsuppressedFailedTests: Collection<TestStaticData>
     ) : Verdict()
 }

subprojects/test-runner/instrumentation-tests/src/main/kotlin/com/avito/instrumentation/internal/finalizer/verdict/VerdictDeterminerImpl.kt

@@ -16,17 +16,18 @@ internal class VerdictDeterminerImpl(
         testResults: Collection<AndroidTest>
     ): Verdict {
 
-        val lostTests = getLostTests(
+        val failedTests = getFailedTests(testResults)
+
+        val notReportedTests = getLostTests(
             initialTestSuite = initialTestSuite,
             testResults = testResults
         )
 
-        val failedTests = getFailedTests(testResults)
-
-        val hasFailedTests = failedTests.isNotEmpty()
+        val unsuppressedFailedTests = mutableListOf<TestStaticData>()
 
         val intermediateResult = when {
-            hasFailedTests -> when {
+
+            failedTests.isNotEmpty() -> when {
                 suppressFailure -> Verdict.Success.Suppressed(
                     testResults = testResults,
                     failedTests = failedTests
@@ -37,11 +38,13 @@ internal class VerdictDeterminerImpl(
 
                     val hasFailedTestsNotMarkedAsFlaky = notFlaky.isNotEmpty()
 
+                    unsuppressedFailedTests.addAll(notFlaky)
+
                     when {
                         hasFailedTestsNotMarkedAsFlaky -> Verdict.Failure(
                             testResults = testResults,
-                            failedTests = notFlaky.toSet(),
-                            lostTests = lostTests
+                            unsuppressedFailedTests = notFlaky.toSet(),
+                            notReportedTests = notReportedTests
                         )
 
                         else -> Verdict.Success.Suppressed(
@@ -50,22 +53,23 @@ internal class VerdictDeterminerImpl(
                         )
                     }
                 }
-                else -> Verdict.Failure(
-                    testResults = testResults,
-                    failedTests = failedTests,
-                    lostTests = lostTests
-                )
+                else -> {
+                    unsuppressedFailedTests.addAll(failedTests)
+                    Verdict.Failure(
+                        testResults = testResults,
+                        unsuppressedFailedTests = failedTests,
+                        notReportedTests = notReportedTests
+                    )
+                }
             }
             else -> Verdict.Success.OK(testResults = testResults)
         }
 
-        val hasLostTests = lostTests.isNotEmpty()
-
-        return if (hasLostTests) {
+        return if (notReportedTests.isNotEmpty()) {
             Verdict.Failure(
                 testResults = testResults,
-                lostTests = lostTests,
-                failedTests = failedTests
+                notReportedTests = notReportedTests,
+                unsuppressedFailedTests = unsuppressedFailedTests
             )
         } else {
             intermediateResult

subprojects/test-runner/instrumentation-tests/src/test/kotlin/com/avito/instrumentation/internal/finalizer/action/WriteTaskVerdictActionTest.kt

@@ -76,20 +76,20 @@ internal class WriteTaskVerdictActionTest {
                 ),
                 failedTest
             ),
-            failedTests = setOf(failedTest),
-            lostTests = emptyList()
+            unsuppressedFailedTests = setOf(failedTest),
+            notReportedTests = emptyList()
         )
         val action = createWriteTaskVerdictAction(tempDir)
         action.action(verdict)
 
         val verdictContent = readFile(tempDir)
 
-        assertThat(verdictContent).contains("Failed. There are 1 not suppressed failed tests")
+        assertThat(verdictContent).contains("Not suppressed failed tests: 1")
         assertThat(verdictContent).contains("com.test.Test2.test API22 FAILED")
     }
 
     @Test
-    fun `file - contains lost message with test - verdict is lost`(@TempDir tempDir: File) {
+    fun `file - contains lost message with test - verdict is failure`(@TempDir tempDir: File) {
         val lostTest = AndroidTest.Lost.createStubInstance(
             testStaticData = TestStaticDataPackage.createStubInstance(
                 name = TestName("com.test.Test2", "test"),
@@ -106,16 +106,46 @@ internal class WriteTaskVerdictActionTest {
                 ),
                 lostTest
             ),
-            failedTests = emptyList(),
-            lostTests = setOf(lostTest)
+            unsuppressedFailedTests = emptyList(),
+            notReportedTests = setOf(lostTest)
         )
         val action = createWriteTaskVerdictAction(tempDir)
         action.action(verdict)
 
         val verdictContent = readFile(tempDir)
 
-        assertThat(verdictContent).contains("Failed. There are 1 not reported tests")
-        assertThat(verdictContent).contains("com.test.Test2.test API22 LOST")
+        assertThat(verdictContent).contains("Not reported tests: 1")
+        assertThat(verdictContent).contains("com.test.Test2.test API22 NOT REPORTED")
+    }
+
+    @Test
+    fun `file - contains lost message without suppressed tests - verdict is failure`(@TempDir tempDir: File) {
+        val lostTest = AndroidTest.Lost.createStubInstance(
+            testStaticData = TestStaticDataPackage.createStubInstance(
+                name = TestName("com.test.Test2", "test"),
+                deviceName = DeviceName("API22")
+            )
+        )
+
+        val verdict = Verdict.Failure(
+            testResults = setOf(
+                AndroidTest.Completed.createStubInstance(
+                    testStaticData = TestStaticDataPackage.createStubInstance(
+                        name = TestName("com.test.Test1", "test")
+                    )
+                ),
+                lostTest
+            ),
+            unsuppressedFailedTests = emptyList(),
+            notReportedTests = setOf(lostTest)
+        )
+        val action = createWriteTaskVerdictAction(tempDir)
+        action.action(verdict)
+
+        val verdictContent = readFile(tempDir)
+
+        assertThat(verdictContent).contains("Not reported tests: 1")
+        assertThat(verdictContent).contains("com.test.Test2.test API22 NOT REPORTED")
     }
 
     private fun createWriteTaskVerdictAction(

subprojects/test-runner/instrumentation-tests/src/test/kotlin/com/avito/instrumentation/internal/finalizer/verdict/VerdictDeterminerImplTest.kt

@@ -56,7 +56,7 @@ internal class VerdictDeterminerImplTest {
         )
 
         assertThat<Verdict.Failure>(verdict) {
-            assertThat(lostTests).contains(lostTest)
+            assertThat(notReportedTests).contains(lostTest)
         }
     }
 
@@ -134,7 +134,7 @@ internal class VerdictDeterminerImplTest {
         )
 
         assertThat<Verdict.Failure>(verdict) {
-            assertThat(failedTests).contains(test)
+            assertThat(unsuppressedFailedTests).contains(test)
         }
     }
 
@@ -204,7 +204,7 @@ internal class VerdictDeterminerImplTest {
         )
 
         assertThat<Verdict.Failure>(verdict) {
-            assertThat(failedTests).contains(test)
+            assertThat(unsuppressedFailedTests).contains(test)
         }
     }
 
@@ -225,7 +225,69 @@ internal class VerdictDeterminerImplTest {
         )
 
         assertThat<Verdict.Failure>(verdict) {
-            assertThat(lostTests).contains(test)
+            assertThat(notReportedTests).contains(test)
+        }
+    }
+
+    @Test
+    fun `verdict - failed and unsuppressedFailedTests is empty - lost and failed reported and fails suppressed`() {
+        val verdictDeterminer = createVerdictDeterminer(suppressFailure = true)
+
+        val lostTest = TestStaticDataPackage.createStubInstance(
+            name = TestName("com.test.Test1.test"),
+            deviceName = DeviceName("DEVICE")
+        )
+
+        val failedTest = TestStaticDataPackage.createStubInstance(
+            name = TestName("com.test.Test2.test"),
+            deviceName = DeviceName("DEVICE")
+        )
+
+        val verdict = verdictDeterminer.determine(
+            initialTestSuite = setOf(
+                lostTest,
+                failedTest
+            ),
+            testResults = listOf(
+                createLostTestExecution(testStaticData = lostTest),
+                createFailedTestExecution(testStaticData = failedTest)
+            )
+        )
+
+        assertThat<Verdict.Failure>(verdict) {
+            assertThat(notReportedTests).contains(lostTest)
+            assertThat(unsuppressedFailedTests).isEmpty()
+        }
+    }
+
+    @Test
+    fun `verdict - failed - lost and failed reported`() {
+        val verdictDeterminer = createVerdictDeterminer(suppressFailure = false)
+
+        val lostTest = TestStaticDataPackage.createStubInstance(
+            name = TestName("com.test.Test1.test"),
+            deviceName = DeviceName("DEVICE")
+        )
+
+        val failedTest = TestStaticDataPackage.createStubInstance(
+            name = TestName("com.test.Test2.test"),
+            deviceName = DeviceName("DEVICE")
+        )
+
+        val verdict = verdictDeterminer.determine(
+            initialTestSuite = setOf(
+                lostTest,
+                failedTest
+            ),
+            testResults = listOf(
+                createLostTestExecution(testStaticData = lostTest),
+                createFailedTestExecution(testStaticData = failedTest)
+            )
+        )
+
+        assertThat<Verdict.Failure>(verdict) {
+            assertThat(notReportedTests).contains(lostTest)
+            assertThat(unsuppressedFailedTests).contains(failedTest)
         }
     }