diff --git a/packages/cli/test/compare/compare-results.test.ts b/packages/cli/test/compare/compare-results.test.ts index 3f66f4cc..47c530c8 100644 --- a/packages/cli/test/compare/compare-results.test.ts +++ b/packages/cli/test/compare/compare-results.test.ts @@ -3,11 +3,11 @@ import { readJsonSync } from "fs-extra"; import { GenerateStats, ITracerBenchTraceResult, - ParsedTitleConfigs, + ParsedTitleConfigs } from "../../src/compare/generate-stats"; import { CompareResults, - ICompareJSONResults, + ICompareJSONResults } from "../../src/compare/compare-results"; import { COMPARE_JSON } from "../test-helpers"; @@ -15,7 +15,7 @@ import { COMPARE_JSON } from "../test-helpers"; const REPORT_TITLES: ParsedTitleConfigs = { servers: [], plotTitle: "Foo Title", - browserVersion: "1.2.3", + browserVersion: "1.2.3" }; const tableDataObj = { @@ -52,7 +52,11 @@ const DEFAULT_REGRESSION_THRESHOLD = 50; const CONTROL_DATA: ITracerBenchTraceResult = readJsonSync(COMPARE_JSON)[0]; const EXPERIMENT_DATA: ITracerBenchTraceResult = readJsonSync(COMPARE_JSON)[1]; const stats = new GenerateStats(CONTROL_DATA, EXPERIMENT_DATA, REPORT_TITLES); -const compareResults = new CompareResults(stats, 20, DEFAULT_REGRESSION_THRESHOLD); +const compareResults = new CompareResults( + stats, + 20, + DEFAULT_REGRESSION_THRESHOLD +); describe("compare-results anyResultsSignificant()", () => { const truthyArr = [true, true, false]; @@ -98,22 +102,42 @@ describe("compare-results allBelowRegressionThreshold()", () => { compareResults.regressionThreshold = DEFAULT_REGRESSION_THRESHOLD; }); it(`regression is below threshold : regressionThresholdStatistic : ci-lower`, () => { - const compareResultsCILower = new CompareResults(stats, 20, 1002, "ci-lower"); + const compareResultsCILower = new CompareResults( + stats, + 20, + 1002, + "ci-lower" + ); const isBelowThresholdCILower = compareResultsCILower.allBelowRegressionThreshold(); expect(isBelowThresholdCILower).to.be.true; }); it(`regression is above threshold : regressionThresholdStatistic : ci-lower`, () => { - const compareResultsCILower = new CompareResults(stats, 20, 995, "ci-lower"); + const compareResultsCILower = new CompareResults( + stats, + 20, + 995, + "ci-lower" + ); const isBelowThresholdCILower = compareResultsCILower.allBelowRegressionThreshold(); expect(isBelowThresholdCILower).to.be.false; }); it(`regression is below threshold : regressionThresholdStatistic : ci-upper`, () => { - const compareResultsCIUpper = new CompareResults(stats, 20, 1100, "ci-upper"); + const compareResultsCIUpper = new CompareResults( + stats, + 20, + 1100, + "ci-upper" + ); const isBelowThresholdCIUpper = compareResultsCIUpper.allBelowRegressionThreshold(); expect(isBelowThresholdCIUpper).to.be.true; }); it(`regression is above threshold : regressionThresholdStatistic : ci-upper`, () => { - const compareResultsCIUpper = new CompareResults(stats, 20, 1000, "ci-upper"); + const compareResultsCIUpper = new CompareResults( + stats, + 20, + 1000, + "ci-upper" + ); const isBelowThresholdCIUpper = compareResultsCIUpper.allBelowRegressionThreshold(); expect(isBelowThresholdCIUpper).to.be.false; }); @@ -139,10 +163,16 @@ describe("compare-results stringifyJSON()", () => { expect(compareJSONResults.isBelowRegressionThreshold).to.be.false; expect(compareJSONResults.regressionThresholdStat).to.eq("estimator"); // lower eg -5 | 2 | 0 - expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMin).to.eq(322.8); + expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMin).to.eq( + 322.77 + ); // middle eg -3 | 5 | 2 - expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMedian).to.eq(323.7); + expect( + compareJSONResults.benchmarkTableData[0].asPercent.percentMedian + ).to.eq(323.68); // upper eg 2 | 10 | 3 - expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMax).to.eq(324.4); + expect(compareJSONResults.benchmarkTableData[0].asPercent.percentMax).to.eq( + 324.38 + ); }); }); diff --git a/packages/cli/test/compare/tb-table.test.ts b/packages/cli/test/compare/tb-table.test.ts index 7112ab5e..b9ae5638 100644 --- a/packages/cli/test/compare/tb-table.test.ts +++ b/packages/cli/test/compare/tb-table.test.ts @@ -68,9 +68,9 @@ describe("tb-table", () => { expect(tableData.length).to.equal(4); // not a regression expect(tableData[0].phaseName).to.equal("stats-high-variance-ms-test"); - expect(tableData[0].asPercent.percentMin).to.equal(-3.332); - expect(tableData[0].asPercent.percentMedian).to.equal(0.07933); - expect(tableData[0].asPercent.percentMax).to.equal(3.768); + expect(tableData[0].asPercent.percentMin).to.equal(-3.33); + expect(tableData[0].asPercent.percentMedian).to.equal(0.08); + expect(tableData[0].asPercent.percentMax).to.equal(3.77); expect(tableData[0].estimatorDelta).to.equal("2ms"); expect(tableData[0].pValue).to.equal(0.967); expect(tableData[0].confidenceInterval[0]).to.equal("-84ms"); @@ -78,7 +78,7 @@ describe("tb-table", () => { expect(tableData[0].isSignificant).to.be.false; // regression expect(tableData[1].phaseName).to.equal("stats-regression-ms-test"); - expect(tableData[1].asPercent.percentMin).to.equal(2081); + expect(tableData[1].asPercent.percentMin).to.equal(2081.14); expect(tableData[1].asPercent.percentMedian).to.equal(2083); expect(tableData[1].asPercent.percentMax).to.equal(2086); expect(tableData[1].estimatorDelta).to.equal("1080ms"); diff --git a/packages/cli/test/fixtures/results/report.json b/packages/cli/test/fixtures/results/report.json index 0e50b531..dd7b6ffb 100644 --- a/packages/cli/test/fixtures/results/report.json +++ b/packages/cli/test/fixtures/results/report.json @@ -1 +1 @@ -{"benchmarkTableData":[{"heading":"Initial Render","phaseName":"duration","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"999ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["996ms","1001ms"],"controlSevenFigureSummary":{"10":307,"25":308,"50":309,"75":311,"90":317,"min":305,"max":434},"experimentSevenFigureSummary":{"10":1303,"25":1307,"50":1309,"75":1311,"90":1320,"min":1299,"max":1328},"asPercent":{"percentMin":322.8,"percentMedian":323.7,"percentMax":324.4}}],"phaseTableData":[{"heading":"Sub Phase of Duration","phaseName":"jquery","isSignificant":false,"pValue":0.4094,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":45,"25":46,"50":46,"75":47,"90":47,"min":40,"max":47},"experimentSevenFigureSummary":{"10":43,"25":45,"50":46,"75":47,"90":48,"min":38,"max":53},"asPercent":{"percentMin":-1.935,"percentMedian":-0.6035,"percentMax":0.7277}},{"heading":"Sub Phase of Duration","phaseName":"ember","isSignificant":false,"pValue":0.8498,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","1ms"],"controlSevenFigureSummary":{"10":128,"25":129,"50":130,"75":131,"90":132,"min":128,"max":135},"experimentSevenFigureSummary":{"10":128,"25":128,"50":130,"75":131,"90":135,"min":127,"max":140},"asPercent":{"percentMin":-0.9999,"percentMedian":-0.07164,"percentMax":0.9136}},{"heading":"Sub Phase of Duration","phaseName":"application","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"1002ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["1001ms","1003ms"],"controlSevenFigureSummary":{"10":24,"25":24,"50":25,"75":26,"90":27,"min":24,"max":28},"experimentSevenFigureSummary":{"10":1025,"25":1026,"50":1026,"75":1029,"90":1031,"min":1025,"max":1032},"asPercent":{"percentMin":4085,"percentMedian":4089,"percentMax":4092}},{"heading":"Sub Phase of Duration","phaseName":"routing","isSignificant":false,"pValue":0.7972,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":18,"25":19,"50":20,"75":20,"90":21,"min":18,"max":27},"experimentSevenFigureSummary":{"10":18,"25":18,"50":20,"75":20,"90":21,"min":18,"max":21},"asPercent":{"percentMin":-4.154,"percentMedian":-0.4203,"percentMax":2.408}},{"heading":"Sub Phase of Duration","phaseName":"transition","isSignificant":false,"pValue":0.4903,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["0ms","0ms"],"controlSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":17},"experimentSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":13},"asPercent":{"percentMin":-2.683,"percentMedian":-0.8065,"percentMax":1.175}},{"heading":"Sub Phase of Duration","phaseName":"render","isSignificant":false,"pValue":0.1636,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":22,"min":20,"max":29},"experimentSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":21,"min":20,"max":23},"asPercent":{"percentMin":-2.588,"percentMedian":-1.122,"percentMax":0.4979}},{"heading":"Sub Phase of Duration","phaseName":"afterRender","isSignificant":true,"pValue":0.00006244,"estimatorDelta":"-2ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-3ms","-1ms"],"controlSevenFigureSummary":{"10":55,"25":56,"50":56,"75":57,"90":61,"min":55,"max":178},"experimentSevenFigureSummary":{"10":53,"25":54,"50":54,"75":55,"90":56,"min":53,"max":60},"asPercent":{"percentMin":-2.232,"percentMedian":-3.598,"percentMax":-4.868}}],"areResultsSignificant":true,"isBelowRegressionThreshold":false,"regressionThresholdStat":"estimator"} \ No newline at end of file +{"benchmarkTableData":[{"heading":"Initial Render","phaseName":"duration","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"999ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["996ms","1001ms"],"controlSevenFigureSummary":{"10":307,"25":308,"50":309,"75":311,"90":317,"min":305,"max":434},"experimentSevenFigureSummary":{"10":1303,"25":1307,"50":1309,"75":1311,"90":1320,"min":1299,"max":1328},"asPercent":{"percentMin":322.77,"percentMedian":323.68,"percentMax":324.38}}],"phaseTableData":[{"heading":"Sub Phase of Duration","phaseName":"jquery","isSignificant":false,"pValue":0.4094,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":45,"25":46,"50":46,"75":47,"90":47,"min":40,"max":47},"experimentSevenFigureSummary":{"10":43,"25":45,"50":46,"75":47,"90":48,"min":38,"max":53},"asPercent":{"percentMin":-1.93,"percentMedian":-0.6,"percentMax":0.73}},{"heading":"Sub Phase of Duration","phaseName":"ember","isSignificant":false,"pValue":0.8498,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","1ms"],"controlSevenFigureSummary":{"10":128,"25":129,"50":130,"75":131,"90":132,"min":128,"max":135},"experimentSevenFigureSummary":{"10":128,"25":128,"50":130,"75":131,"90":135,"min":127,"max":140},"asPercent":{"percentMin":-1,"percentMedian":-0.07,"percentMax":0.91}},{"heading":"Sub Phase of Duration","phaseName":"application","isSignificant":true,"pValue":6.796e-8,"estimatorDelta":"1002ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["1001ms","1003ms"],"controlSevenFigureSummary":{"10":24,"25":24,"50":25,"75":26,"90":27,"min":24,"max":28},"experimentSevenFigureSummary":{"10":1025,"25":1026,"50":1026,"75":1029,"90":1031,"min":1025,"max":1032},"asPercent":{"percentMin":4084.86,"percentMedian":4088.69,"percentMax":4091.59}},{"heading":"Sub Phase of Duration","phaseName":"routing","isSignificant":false,"pValue":0.7972,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":18,"25":19,"50":20,"75":20,"90":21,"min":18,"max":27},"experimentSevenFigureSummary":{"10":18,"25":18,"50":20,"75":20,"90":21,"min":18,"max":21},"asPercent":{"percentMin":-4.15,"percentMedian":-0.42,"percentMax":2.41}},{"heading":"Sub Phase of Duration","phaseName":"transition","isSignificant":false,"pValue":0.4903,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["0ms","0ms"],"controlSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":17},"experimentSevenFigureSummary":{"10":11,"25":11,"50":11,"75":12,"90":12,"min":11,"max":13},"asPercent":{"percentMin":-2.68,"percentMedian":-0.81,"percentMax":1.17}},{"heading":"Sub Phase of Duration","phaseName":"render","isSignificant":false,"pValue":0.1636,"estimatorDelta":"0ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-1ms","0ms"],"controlSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":22,"min":20,"max":29},"experimentSevenFigureSummary":{"10":20,"25":20,"50":20,"75":21,"90":21,"min":20,"max":23},"asPercent":{"percentMin":-2.59,"percentMedian":-1.12,"percentMax":0.5}},{"heading":"Sub Phase of Duration","phaseName":"afterRender","isSignificant":true,"pValue":0.00006244,"estimatorDelta":"-2ms","controlSampleCount":20,"experimentSampleCount":20,"confidenceInterval":["-3ms","-1ms"],"controlSevenFigureSummary":{"10":55,"25":56,"50":56,"75":57,"90":61,"min":55,"max":178},"experimentSevenFigureSummary":{"10":53,"25":54,"50":54,"75":55,"90":56,"min":53,"max":60},"asPercent":{"percentMin":-4.87,"percentMedian":-3.6,"percentMax":-2.23}}],"areResultsSignificant":true,"isBelowRegressionThreshold":false,"regressionThresholdStat":"estimator"} \ No newline at end of file diff --git a/packages/stats/test/stats.test.ts b/packages/stats/test/stats.test.ts index 9b24bfc2..2ac92fba 100644 --- a/packages/stats/test/stats.test.ts +++ b/packages/stats/test/stats.test.ts @@ -76,14 +76,14 @@ describe('stats', () => { expect(stats.confidenceInterval.U).to.equal(625); expect(stats.confidenceInterval.asPercent.percentMin).to.equal(-2086.2); expect(stats.confidenceInterval.asPercent.percentMedian).to.equal(-2083.26); - expect(stats.confidenceInterval.asPercent.percentMax).to.equal(-2081.14); + expect(stats.confidenceInterval.asPercent.percentMax).to.equal(-2081); // MS expect(statsMS.confidenceInterval.asPercent.percentMin).to.equal(-2086.2); expect(statsMS.confidenceInterval.asPercent.percentMedian).to.equal( -2083.26 ); - expect(statsMS.confidenceInterval.asPercent.percentMax).to.equal(-2081.14); + expect(statsMS.confidenceInterval.asPercent.percentMax).to.equal(-2081); // HIGH VARIANCE MS expect(statsHighVarianceMS.confidenceInterval.min).to.equal(-95);