test(cli, stats): update failing tests from rounding change

TracerBench · Apr 20, 2021 · e070b4b · e070b4b
1 parent a70ad29
commit e070b4b
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 18 deletions.
diff --git a/packages/cli/test/compare/compare-results.test.ts b/packages/cli/test/compare/compare-results.test.ts
@@ -3,19 +3,19 @@ import { readJsonSync } from "fs-extra";
 import {
   GenerateStats,
   ITracerBenchTraceResult,
-  ParsedTitleConfigs,
+  ParsedTitleConfigs
 } from "../../src/compare/generate-stats";
 import {
   CompareResults,
-  ICompareJSONResults,
+  ICompareJSONResults
 } from "../../src/compare/compare-results";
 
 import { COMPARE_JSON } from "../test-helpers";
 
 const REPORT_TITLES: ParsedTitleConfigs = {
   servers: [],
   plotTitle: "Foo Title",
-  browserVersion: "1.2.3",
+  browserVersion: "1.2.3"
 };
 
 const tableDataObj = {
@@ -52,7 +52,11 @@ const DEFAULT_REGRESSION_THRESHOLD = 50;
 const CONTROL_DATA: ITracerBenchTraceResult = readJsonSync(COMPARE_JSON)[0];
 const EXPERIMENT_DATA: ITracerBenchTraceResult = readJsonSync(COMPARE_JSON)[1];
 const stats = new GenerateStats(CONTROL_DATA, EXPERIMENT_DATA, REPORT_TITLES);
-const compareResults = new CompareResults(stats, 20, DEFAULT_REGRESSION_THRESHOLD);
+const compareResults = new CompareResults(
+  stats,
+  20,
+  DEFAULT_REGRESSION_THRESHOLD
+);
 
 describe("compare-results anyResultsSignificant()", () => {
   const truthyArr = [true, true, false];
@@ -98,22 +102,42 @@ describe("compare-results allBelowRegressionThreshold()", () => {
     compareResults.regressionThreshold = DEFAULT_REGRESSION_THRESHOLD;
   });
   it(`regression is below threshold : regressionThresholdStatistic : ci-lower`, () => {
-    const compareResultsCILower = new CompareResults(stats, 20, 1002, "ci-lower");
+    const compareResultsCILower = new CompareResults(
+      stats,
+      20,
+      1002,
+      "ci-lower"
+    );
     const isBelowThresholdCILower = compareResultsCILower.allBelowRegressionThreshold();
     expect(isBelowThresholdCILower).to.be.true;
   });
   it(`regression is above threshold : regressionThresholdStatistic : ci-lower`, () => {
-    const compareResultsCILower = new CompareResults(stats, 20, 995, "ci-lower");
+    const compareResultsCILower = new CompareResults(
+      stats,
+      20,
+      995,
+      "ci-lower"
+    );
     const isBelowThresholdCILower = compareResultsCILower.allBelowRegressionThreshold();
     expect(isBelowThresholdCILower).to.be.false;
   });
   it(`regression is below threshold : regressionThresholdStatistic : ci-upper`, () => {
-    const compareResultsCIUpper = new CompareResults(stats, 20, 1100, "ci-upper");
+    const compareResultsCIUpper = new CompareResults(
+      stats,
+      20,
+      1100,
+      "ci-upper"
+    );
     const isBelowThresholdCIUpper = compareResultsCIUpper.allBelowRegressionThreshold();
     expect(isBelowThresholdCIUpper).to.be.true;
   });
   it(`regression is above threshold : regressionThresholdStatistic : ci-upper`, () => {
-    const compareResultsCIUpper = new CompareResults(stats, 20, 1000, "ci-upper");
+    const compareResultsCIUpper = new CompareResults(
+      stats,
+      20,
+      1000,
+      "ci-upper"
+    );
     const isBelowThresholdCIUpper = compareResultsCIUpper.allBelowRegressionThreshold();
     expect(isBelowThresholdCIUpper).to.be.false;
   });
@@ -139,10 +163,16 @@ describe("compare-results stringifyJSON()", () => {
     expect(compareJSONResults.isBelowRegressionThreshold).to.be.false;
     expect(compareJSONResults.regressionThresholdStat).to.eq("estimator");
     // lower eg -5 | 2 | 0
-    expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMin).to.eq(322.8);
+    expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMin).to.eq(
+      322.77
+    );
     // middle eg -3 | 5 | 2
-    expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMedian).to.eq(323.7);
+    expect(
+      compareJSONResults.benchmarkTableData[0].asPercent.percentMedian
+    ).to.eq(323.68);
     // upper eg 2 | 10 | 3
-    expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMax).to.eq(324.4);
+    expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMax).to.eq(
+      324.38
+    );
   });
 });
diff --git a/packages/cli/test/compare/tb-table.test.ts b/packages/cli/test/compare/tb-table.test.ts
@@ -68,17 +68,17 @@ describe("tb-table", () => {
     expect(tableData.length).to.equal(4);
     // not a regression
     expect(tableData[0].phaseName).to.equal("stats-high-variance-ms-test");
-    expect(tableData[0].asPercent.percentMin).to.equal(-3.332);
-    expect(tableData[0].asPercent.percentMedian).to.equal(0.07933);
-    expect(tableData[0].asPercent.percentMax).to.equal(3.768);
+    expect(tableData[0].asPercent.percentMin).to.equal(-3.33);
+    expect(tableData[0].asPercent.percentMedian).to.equal(0.08);
+    expect(tableData[0].asPercent.percentMax).to.equal(3.77);
     expect(tableData[0].estimatorDelta).to.equal("2ms");
     expect(tableData[0].pValue).to.equal(0.967);
     expect(tableData[0].confidenceInterval[0]).to.equal("-84ms");
     expect(tableData[0].confidenceInterval[1]).to.equal("95ms");
     expect(tableData[0].isSignificant).to.be.false;
     // regression
     expect(tableData[1].phaseName).to.equal("stats-regression-ms-test");
-    expect(tableData[1].asPercent.percentMin).to.equal(2081);
+    expect(tableData[1].asPercent.percentMin).to.equal(2081.14);
     expect(tableData[1].asPercent.percentMedian).to.equal(2083);
     expect(tableData[1].asPercent.percentMax).to.equal(2086);
     expect(tableData[1].estimatorDelta).to.equal("1080ms");

diff --git a/packages/cli/test/fixtures/results/report.json b/packages/cli/test/fixtures/results/report.json
@@ -1 +1 @@
-{"benchmarkTableData":[{"heading":"Initial Render","phaseName":"duration","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"999ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["996ms","1001ms"],"controlSevenFigureSummary":{"10":307,"25":308,"50":309,"75":311,"90":317,"min":305,"max":434},"experimentSevenFigureSummary":{"10":1303,"25":1307,"50":1309,"75":1311,"90":1320,"min":1299,"max":1328},"asPercent":{"percentMin":322.8,"percentMedian":323.7,"percentMax":324.4}}],"phaseTableData":[{"heading":"Sub Phase of Duration","phaseName":"jquery","isSignificant":false,"pValue":0.4094,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":45,"25":46,"50":46,"75":47,"90":47,"min":40,"max":47},"experimentSevenFigureSummary":{"10":43,"25":45,"50":46,"75":47,"90":48,"min":38,"max":53},"asPercent":{"percentMin":-1.935,"percentMedian":-0.6035,"percentMax":0.7277}},{"heading":"Sub Phase of Duration","phaseName":"ember","isSignificant":false,"pValue":0.8498,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","1ms"],"controlSevenFigureSummary":{"10":128,"25":129,"50":130,"75":131,"90":132,"min":128,"max":135},"experimentSevenFigureSummary":{"10":128,"25":128,"50":130,"75":131,"90":135,"min":127,"max":140},"asPercent":{"percentMin":-0.9999,"percentMedian":-0.07164,"percentMax":0.9136}},{"heading":"Sub Phase of Duration","phaseName":"application","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"1002ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["1001ms","1003ms"],"controlSevenFigureSummary":{"10":24,"25":24,"50":25,"75":26,"90":27,"min":24,"max":28},"experimentSevenFigureSummary":{"10":1025,"25":1026,"50":1026,"75":1029,"90":1031,"min":1025,"max":1032},"asPercent":{"percentMin":4085,"percentMedian":4089,"percentMax":4092}},{"heading":"Sub Phase of Duration","phaseName":"routing","isSignificant":false,"pValue":0.7972,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":18,"25":19,"50":20,"75":20,"90":21,"min":18,"max":27},"experimentSevenFigureSummary":{"10":18,"25":18,"50":20,"75":20,"90":21,"min":18,"max":21},"asPercent":{"percentMin":-4.154,"percentMedian":-0.4203,"percentMax":2.408}},{"heading":"Sub Phase of Duration","phaseName":"transition","isSignificant":false,"pValue":0.4903,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["0ms","0ms"],"controlSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":17},"experimentSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":13},"asPercent":{"percentMin":-2.683,"percentMedian":-0.8065,"percentMax":1.175}},{"heading":"Sub Phase of Duration","phaseName":"render","isSignificant":false,"pValue":0.1636,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":22,"min":20,"max":29},"experimentSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":21,"min":20,"max":23},"asPercent":{"percentMin":-2.588,"percentMedian":-1.122,"percentMax":0.4979}},{"heading":"Sub Phase of Duration","phaseName":"afterRender","isSignificant":true,"pValue":0.00006244,"estimatorDelta":"-2ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-3ms","-1ms"],"controlSevenFigureSummary":{"10":55,"25":56,"50":56,"75":57,"90":61,"min":55,"max":178},"experimentSevenFigureSummary":{"10":53,"25":54,"50":54,"75":55,"90":56,"min":53,"max":60},"asPercent":{"percentMin":-2.232,"percentMedian":-3.598,"percentMax":-4.868}}],"areResultsSignificant":true,"isBelowRegressionThreshold":false,"regressionThresholdStat":"estimator"}
+{"benchmarkTableData":[{"heading":"Initial Render","phaseName":"duration","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"999ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["996ms","1001ms"],"controlSevenFigureSummary":{"10":307,"25":308,"50":309,"75":311,"90":317,"min":305,"max":434},"experimentSevenFigureSummary":{"10":1303,"25":1307,"50":1309,"75":1311,"90":1320,"min":1299,"max":1328},"asPercent":{"percentMin":322.77,"percentMedian":323.68,"percentMax":324.38}}],"phaseTableData":[{"heading":"Sub Phase of Duration","phaseName":"jquery","isSignificant":false,"pValue":0.4094,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":45,"25":46,"50":46,"75":47,"90":47,"min":40,"max":47},"experimentSevenFigureSummary":{"10":43,"25":45,"50":46,"75":47,"90":48,"min":38,"max":53},"asPercent":{"percentMin":-1.93,"percentMedian":-0.6,"percentMax":0.73}},{"heading":"Sub Phase of Duration","phaseName":"ember","isSignificant":false,"pValue":0.8498,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","1ms"],"controlSevenFigureSummary":{"10":128,"25":129,"50":130,"75":131,"90":132,"min":128,"max":135},"experimentSevenFigureSummary":{"10":128,"25":128,"50":130,"75":131,"90":135,"min":127,"max":140},"asPercent":{"percentMin":-1,"percentMedian":-0.07,"percentMax":0.91}},{"heading":"Sub Phase of Duration","phaseName":"application","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"1002ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["1001ms","1003ms"],"controlSevenFigureSummary":{"10":24,"25":24,"50":25,"75":26,"90":27,"min":24,"max":28},"experimentSevenFigureSummary":{"10":1025,"25":1026,"50":1026,"75":1029,"90":1031,"min":1025,"max":1032},"asPercent":{"percentMin":4084.86,"percentMedian":4088.69,"percentMax":4091.59}},{"heading":"Sub Phase of Duration","phaseName":"routing","isSignificant":false,"pValue":0.7972,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":18,"25":19,"50":20,"75":20,"90":21,"min":18,"max":27},"experimentSevenFigureSummary":{"10":18,"25":18,"50":20,"75":20,"90":21,"min":18,"max":21},"asPercent":{"percentMin":-4.15,"percentMedian":-0.42,"percentMax":2.41}},{"heading":"Sub Phase of Duration","phaseName":"transition","isSignificant":false,"pValue":0.4903,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["0ms","0ms"],"controlSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":17},"experimentSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":13},"asPercent":{"percentMin":-2.68,"percentMedian":-0.81,"percentMax":1.17}},{"heading":"Sub Phase of Duration","phaseName":"render","isSignificant":false,"pValue":0.1636,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":22,"min":20,"max":29},"experimentSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":21,"min":20,"max":23},"asPercent":{"percentMin":-2.59,"percentMedian":-1.12,"percentMax":0.5}},{"heading":"Sub Phase of Duration","phaseName":"afterRender","isSignificant":true,"pValue":0.00006244,"estimatorDelta":"-2ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-3ms","-1ms"],"controlSevenFigureSummary":{"10":55,"25":56,"50":56,"75":57,"90":61,"min":55,"max":178},"experimentSevenFigureSummary":{"10":53,"25":54,"50":54,"75":55,"90":56,"min":53,"max":60},"asPercent":{"percentMin":-4.87,"percentMedian":-3.6,"percentMax":-2.23}}],"areResultsSignificant":true,"isBelowRegressionThreshold":false,"regressionThresholdStat":"estimator"}
diff --git a/packages/stats/test/stats.test.ts b/packages/stats/test/stats.test.ts
@@ -76,14 +76,14 @@ describe('stats', () => {
     expect(stats.confidenceInterval.U).to.equal(625);
     expect(stats.confidenceInterval.asPercent.percentMin).to.equal(-2086.2);
     expect(stats.confidenceInterval.asPercent.percentMedian).to.equal(-2083.26);
-    expect(stats.confidenceInterval.asPercent.percentMax).to.equal(-2081.14);
+    expect(stats.confidenceInterval.asPercent.percentMax).to.equal(-2081);
 
     // MS
     expect(statsMS.confidenceInterval.asPercent.percentMin).to.equal(-2086.2);
     expect(statsMS.confidenceInterval.asPercent.percentMedian).to.equal(
       -2083.26
     );
-    expect(statsMS.confidenceInterval.asPercent.percentMax).to.equal(-2081.14);
+    expect(statsMS.confidenceInterval.asPercent.percentMax).to.equal(-2081);
 
     // HIGH VARIANCE MS
     expect(statsHighVarianceMS.confidenceInterval.min).to.equal(-95);