relax performance targets to ~10%, improve perf failure message

lostmsu · lostmsu · commit a6c55d4b66bf · 2020-01-31T12:33:08.000-08:00
diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs
@@ -30,14 +30,14 @@ public void SetUp()
         public void ReadInt64Property()
         {
             double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
-            Assert.LessOrEqual(optimisticPerfRatio, 0.68);
+            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66);
         }
 
         [Test]
         public void WriteInt64Property()
         {
             double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
-            Assert.LessOrEqual(optimisticPerfRatio, 0.67);
+            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64);
         }
 
         static double GetOptimisticPerfRatio(
@@ -61,5 +61,14 @@ static double GetOptimisticPerfRatio(
         }
 
         public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+
+        public static void AssertPerformanceIsBetterOrSame(
+            double actual, double target,
+            double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {
+            double threshold = target * wiggleRoom;
+            Assert.LessOrEqual(actual, threshold,
+                $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"
+                + ": perf result is higher than the failure threshold.");
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -30,14 +30,14 @@ public void SetUp()`
`30`	`30`	`public void ReadInt64Property()`
`31`	`31`	`{`
`32`	`32`	`double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);`
`33`		`- Assert.LessOrEqual(optimisticPerfRatio, 0.68);`
	`33`	`+ AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66);`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`[Test]`
`37`	`37`	`public void WriteInt64Property()`
`38`	`38`	`{`
`39`	`39`	`double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);`
`40`		`- Assert.LessOrEqual(optimisticPerfRatio, 0.67);`
	`40`	`+ AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64);`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`static double GetOptimisticPerfRatio(`
`@@ -61,5 +61,14 @@ static double GetOptimisticPerfRatio(`
`61`	`61`	`}`
`62`	`62`
`63`	`63`	`public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);`
	`64`	`+`
	`65`	`+ public static void AssertPerformanceIsBetterOrSame(`
	`66`	`+ double actual, double target,`
	`67`	`+ double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {`
	`68`	`+ double threshold = target * wiggleRoom;`
	`69`	`+ Assert.LessOrEqual(actual, threshold,`
	`70`	`+ $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"`
	`71`	`+ + ": perf result is higher than the failure threshold.");`
	`72`	`+ }`
`64`	`73`	`}`
`65`	`74`	`}`