Skip to content

Commit

Permalink
PER-1548 Add possibility to give individual thresholds for tests (#208)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fat-fellow
fat-fellow authored Feb 25, 2020
1 parent 65b9c1c commit d5e1b33
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 24 deletions.
27 changes: 21 additions & 6 deletions ...jects/gradle/performance/src/main/kotlin/com/avito/performance/PerformanceTestReporter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ internal enum class Metric(
FPS(
"FPS",
shouldBeLess = false,
diffThreshold = 3.0,
diffThreshold = 2.0,
isBlocker = true
),
PROPER_FRAMES_PERCENT(
Expand All @@ -74,7 +74,7 @@ internal enum class Metric(
ACTIVITY_STARTUP_TIME(
"ActivityStartupTime",
shouldBeLess = true,
diffThreshold = 150.0,
diffThreshold = 200.0,
isBlocker = true
),
JUNKY_FRAME_MAX_DURATION(
Expand Down Expand Up @@ -113,17 +113,31 @@ internal fun ComparedTest.Comparison.failed() =
} else {
val greaterThanExpected = it.greaterThanExpected(metric)
val lessThanExpected = it.lessThanExpected(metric)
val significantThreshold = significantThreshold(
value = it.value.meanDiff,
threshold = metric.diffThreshold
)
val significantThreshold = isSignificant(it.value, metric)
val blocker = metric.isBlocker
((lessThanExpected || greaterThanExpected)
&& significantThreshold
&& blocker)
}
}

private fun isSignificant(
series: ComparedTest.Series,
metric: Metric
): Boolean {
return if (abs(series.thresholdStatic) < EPSILON) {
significantThreshold(
value = series.meanDiff,
threshold = metric.diffThreshold
)
} else {
significantThreshold(
value = series.meanDiff,
threshold = series.thresholdStatic
)
}
}

internal fun ComparedTest.Comparison.performedMuchBetterThanUsual() =
this.series
.filter {
Expand All @@ -146,3 +160,4 @@ internal fun ComparedTest.Comparison.performedMuchBetterThanUsual() =

private const val NEW_LINE = "\n"
private const val PERFORMER = "Performer"
private const val EPSILON = 1e-9
6 changes: 3 additions & 3 deletions ...gradle/performance/src/main/kotlin/com/avito/performance/stats/comparison/ComparedTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ internal interface ComparedTest {
val currentSampleIs: State,
val statistic: Double,
val pValue: Double,
//current-previous
val meanDiff: Double,
val threshold: Double
val meanDiff: Double, //current-previous
val threshold: Double,
val thresholdStatic: Double //overrides threshold for particular tests in mongo
)

enum class State {
Expand Down
27 changes: 14 additions & 13 deletions ...cts/gradle/performance/src/test/kotlin/com/avito/performance/PerformanceComparatorTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ internal class PerformanceComparatorTest {
Assertions.assertTrue(outputFile.exists()) { "current-runs.txt should exist!" }

Truth.assertThat(outputFile.readText()).contains(
("""[{"testName":"testname0\"","id":"0","series":{"testname0\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}},
|{"testName":"testname1\"","id":"1","series":{"testname1\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}},
|{"testName":"testname2\"","id":"2","series":{"testname2\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}},
|{"testName":"testname3\"","id":"3","series":{"testname3\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}}]"""
("""[{"testName":"testname0\"","id":"0","series":{"testname0\"_fps":{"significance":0.0,"currentSampleIs":
|"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}},
|{"testName":"testname1\"","id":"1","series":{"testname1\"_fps":{"significance":0.0,"currentSampleIs":
|"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}},
|{"testName":"testname2\"","id":"2","series":{"testname2\"_fps":{"significance":0.0,"currentSampleIs":
|"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}},
|{"testName":"testname3\"","id":"3","series":{"testname3\"_fps":{"significance":0.0,"currentSampleIs":
|"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}}]"""
.trimMargin())
.trimIndent()
.replace("\n", "")
Expand All @@ -61,10 +61,10 @@ internal class PerformanceComparatorTest {
Assertions.assertTrue(outputFile.exists()) { "current-runs.txt should exist" }

Truth.assertThat(outputFile.readText()).contains(
("""[{"testName":"testname0\"","id":"0","series":{"testname0\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}},
|{"testName":"testname1\"","id":"1","series":{"testname1\"_fps":{"significance":0.0,
|"currentSampleIs":"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0}}}]"""
("""[{"testName":"testname0\"","id":"0","series":{"testname0\"_fps":{"significance":0.0,"currentSampleIs"
|:"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}},
|{"testName":"testname1\"","id":"1","series":{"testname1\"_fps":{"significance":0.0,"currentSampleIs":
|"same","statistic":0.0,"pValue":1.0,"meanDiff":2.0,"threshold":100.0,"thresholdStatic":0.0}}}]"""
.trimMargin())
.trimIndent()
.replace("\n", "")
Expand Down Expand Up @@ -112,7 +112,8 @@ internal class PerformanceComparatorTest {
0.0,
1.0,
2.0,
threshold = 100.0
threshold = 100.0,
thresholdStatic = 0.0
)
}.toMap()
)
Expand Down
3 changes: 2 additions & 1 deletion subprojects/gradle/performance/src/test/kotlin/com/avito/performance/stats/StatsApiTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ internal class StatsApiTest {
statistic = 0.0,
pValue = 0.2,
meanDiff = 2.0,
threshold = 100.0
threshold = 100.0,
thresholdStatic = 0.0
)
)
)
Expand Down
40 changes: 39 additions & 1 deletion ...ance/src/test/kotlin/com/avito/performance/stats/comparison/ComparedTestComparisonTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,42 @@ internal class ComparedTestComparisonTest {
.hasSize(1)
}

@Test
fun `failed - returns 0 - if less than zero and should be greater and less than static threshold`() {
assertThat(
givenTestComparison(
currentSampleIs = LESS,
meanDiff = -1.0,
metric = "FPS",
thresholdStatic = 2.0
).failed())
.hasSize(0)
}

@Test
fun `failed - returns 0 - if greater than zero and should be greater and greater than static threshold`() {
assertThat(
givenTestComparison(
currentSampleIs = GREATER,
meanDiff = 1.0,
metric = "FPS",
thresholdStatic = 0.5
).failed())
.hasSize(0)
}

@Test
fun `failed - returns 1 - if less than zero and should be greater and greater than static threshold`() {
assertThat(
givenTestComparison(
currentSampleIs = LESS,
meanDiff = -1.0,
metric = "FPS",
thresholdStatic = 0.5
).failed())
.hasSize(1)
}

@Test
fun `failed - returns 1 - if less than zero and should be greater and greater than threshold`() {
assertThat(
Expand Down Expand Up @@ -187,6 +223,7 @@ internal class ComparedTestComparisonTest {
//current-previous
meanDiff: Double = 0.0,
threshold: Double = 0.0,
thresholdStatic: Double = 0.0,
metric: String = "FPS"
): Comparison {
return Comparison(
Expand All @@ -198,7 +235,8 @@ internal class ComparedTestComparisonTest {
statistic,
pValue,
meanDiff,
threshold
threshold,
thresholdStatic
)
)
)
Expand Down

0 comments on commit d5e1b33

Please sign in to comment.