diff --git a/README.md b/README.md index f2fdd6f3..55f43129 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Version](https://img.shields.io/npm/v/tracerbench.svg)](https://npmjs.org/package/tracerbench) [![License](https://img.shields.io/npm/l/tracerbench.svg)](https://github.com/TracerBench/tracerbench/blob/master/package.json) -As it pertains to data analysis, intuition almost always leads us astray. We see patterns in random data and jump to unwarranted conclusions. We need a guide. One that uses statistical rigor so that we can make valid conclusions based upon the data. TracerBench aims to be that guide. +As it pertains to data analysis, intuition almost always leads us astray. We see patterns in random data and jump to unwarranted conclusions. We need a guide. One that uses statistical rigor so that we can make valid conclusions based upon the data. TracerBench aims to be that guide. TracerBench is a controlled performance benchmarking tool for web applications. Providing clear, actionable and usable insights into performance deltas. By extracting metrics around response, animation, idle, and load through automated chrome traces and controlling that each of our samples is independent. TracerBench is able to provide low variance and reproducible performance data. TracerBench results are packageable and shareable allowing for replicated peer review. @@ -16,8 +16,8 @@ The current approach for performance analysis for developers is running a single TracerBench has been greatly inspired by the Chromium benchmark tool [Telemetry](https://github.com/catapult-project/catapult/blob/master/telemetry/docs/run_benchmarks_locally.md). - ## How does TracerBench compare to [Lighthouse](https://github.com/GoogleChrome/lighthouse)? + When comparing TracerBench to the most popular tool Chrome Developer Tools Lighthouse. The primary difference is TracerBench is focused on getting a low variance for a metric across many samples versus getting a hard to replicate “Lighthouse performance report”. Lighthouse is essentially a black-box, with developers unable to customize performance parameters in-depth and lacking proper statistical rigor. TracerBench on the other hand, can be highly instrumented, provides statistical rigor and adequate sampling of data. Additionally, TracerBench instrumentation has minimal impact on the overhead of the application; as such TracerBench instrumentation can be "checked-in" and left in your application without worry of negative performance impacts. # User-Stories @@ -51,13 +51,11 @@ Assuming the TracerBench-CLI is globally [installed](https://github.com/TracerBe 1. Start by having TracerBench record a HAR: ```console -$ tracerbench create-archive --url http://localhost:8000 -... +$ tracerbench record-har --url http://localhost:8000 --cookies ✔ DevTools listening on ws://
✔ { timestamp: 241968.79908 } ✔ HAR successfully generated from http://localhost:8000 and available here: ./trace.har -✔ Cookies successfully generated and available here: ./cookies.json ``` 2. Now have TracerBench record a Trace of that HAR: @@ -109,7 +107,7 @@ In your app you must place a marker to let TracerBench know that you are done re function renderMyApp() { // basic "web application" // literally an app with a single empty p tag - const p = document.createElement("p"); + const p = document.createElement('p'); document.body.appendChild(p); } @@ -164,8 +162,8 @@ const browser = { '--v8-cache-options=none', '--disable-cache', '--disable-v8-idle-tasks', - '--crash-dumps-dir=./tmp' - ] + '--crash-dumps-dir=./tmp', + ], }; // name, url, markers and browser are all required options @@ -178,7 +176,7 @@ const control = new InitialRenderBenchmark({ markers, browser, // location to save only the control trace to - saveTraces: () => `./control-trace.json` + saveTraces: () => `./control-trace.json`, }); const experiment = new InitialRenderBenchmark({ @@ -187,7 +185,7 @@ const experiment = new InitialRenderBenchmark({ markers, browser, // location to save only the experiment trace to - saveTraces: () => `./experiment-trace.json` + saveTraces: () => `./experiment-trace.json`, }); // the runner uses the config of each benchmark to test against @@ -244,6 +242,7 @@ When running the TracerBench-CLI `compare` command, on a successful trace a stdo TracerBench also exposes an explicit `tracerbench report` command that takes a path to the folder containing your "trace-results.json" file and will create a PDF and HTML report. ### Understanding The Box-Plot Results + ![box-plot-results](https://github.com/TracerBench/tracerbench/blob/master/docs/box-plot-transparent.png) --- diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 76c529f6..48a5a922 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -2,7 +2,6 @@ cookies.json *tracerbench-results* *trace.json -*trace.har *tsconfig.tsbuildinfo *tb-tmp* *scratch* diff --git a/packages/cli/README.md b/packages/cli/README.md index 054e75c2..7eab97a3 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -194,40 +194,14 @@ s?: TRACE_EVENT_SCOPE; -- [`tracerbench`](#tracerbench-) - [`tracerbench compare`](#tracerbench-compare) - [`tracerbench compare:analyze RESULTSFILE`](#tracerbench-compareanalyze-resultsfile) -- [`tracerbench create-archive`](#tracerbench-create-archive) - [`tracerbench help [COMMAND]`](#tracerbench-help-command) - [`tracerbench marker-timings`](#tracerbench-marker-timings) +- [`tracerbench record-har`](#tracerbench-record-har) - [`tracerbench report`](#tracerbench-report) - [`tracerbench trace`](#tracerbench-trace) -## `tracerbench` - -Creates an automated HAR file from a URL. - -``` -USAGE - $ tracerbench - -OPTIONS - --browserArgs=browserArgs - (required) [default: - --crash-dumps-dir=./tmp,--disable-background-timer-throttling,--disable-dev-shm-usage,--disable-cache,--disable-v8-i - dle-tasks,--disable-breakpad,--disable-notifications,--disable-hang-monitor,--safebrowsing-disable-auto-update,--ign - ore-certificate-errors,--v8-cache-options=none] (Default Recommended) Additional chrome flags for the TracerBench - render benchmark. TracerBench includes many non-configurable defaults in this category. - - --tbResultsFolder=tbResultsFolder - (required) [default: ./tracerbench-results] The output folder path for all tracerbench results - - --url=url - (required) [default: http://localhost:8000/] URL to visit for create-archive, timings & trace commands -``` - -_See code: [dist/src/commands/index.ts](https://github.com/TracerBench/tracerbench/tree/master/packages/cli/blob/v2.3.0/dist/src/commands/index.ts)_ - ## `tracerbench compare` Compare the performance delta between an experiment and control @@ -324,33 +298,6 @@ OPTIONS tracerbench results ``` -_See code: [dist/src/commands/compare/analyze.ts](https://github.com/TracerBench/tracerbench/tree/master/packages/cli/blob/v2.3.0/dist/src/commands/compare/analyze.ts)_ - -## `tracerbench create-archive` - -Creates an automated HAR file from a URL. - -``` -USAGE - $ tracerbench create-archive - -OPTIONS - --browserArgs=browserArgs - (required) [default: - --crash-dumps-dir=./tmp,--disable-background-timer-throttling,--disable-dev-shm-usage,--disable-cache,--disable-v8-i - dle-tasks,--disable-breakpad,--disable-notifications,--disable-hang-monitor,--safebrowsing-disable-auto-update,--ign - ore-certificate-errors,--v8-cache-options=none] (Default Recommended) Additional chrome flags for the TracerBench - render benchmark. TracerBench includes many non-configurable defaults in this category. - - --tbResultsFolder=tbResultsFolder - (required) [default: ./tracerbench-results] The output folder path for all tracerbench results - - --url=url - (required) [default: http://localhost:8000/] URL to visit for create-archive, timings & trace commands -``` - -_See code: [dist/src/commands/create-archive.ts](https://github.com/TracerBench/tracerbench/tree/master/packages/cli/blob/v2.3.0/dist/src/commands/create-archive.ts)_ - ## `tracerbench help [COMMAND]` display help for tracerbench @@ -384,8 +331,27 @@ OPTIONS --traceFrame=traceFrame Specify a trace insights frame - --url=url (required) [default: http://localhost:8000/] URL to visit for create-archive, - timings & trace commands + --url=url (required) [default: http://localhost:8000/] URL to visit for record-har, timings & + trace commands +``` + +## `tracerbench record-har` + +Generates a HAR file from a URL. + +``` +USAGE + $ tracerbench record-har + +OPTIONS + --cookiespath=cookiespath (required) The path to a JSON file containing cookies to authenticate against the + correlated URL + + --dest=dest (required) The destination path for the generated file + + --filename=filename (required) [default: tracerbench] The filename for the generated file + + --url=url (required) URL to visit for record-har, timings & trace commands ``` _See code: [dist/src/commands/marker-timings.ts](https://github.com/TracerBench/tracerbench/tree/master/packages/cli/blob/v2.3.0/dist/src/commands/marker-timings.ts)_ @@ -417,9 +383,15 @@ USAGE $ tracerbench trace OPTIONS + --cookiespath=cookiespath + (required) The path to a JSON file containing cookies to authenticate against the correlated URL + --cpuThrottleRate=cpuThrottleRate (required) [default: 2] CPU throttle multiplier + --harpath=harpath + (required) The path to the HTTP Archive File (HAR) + --insights Analyze insights from command. @@ -436,7 +408,7 @@ OPTIONS (required) [default: ./tracerbench-results] The output folder path for all tracerbench results --url=url - (required) [default: http://localhost:8000/] URL to visit for create-archive, timings & trace commands + (required) [default: http://localhost:8000/] URL to visit for record-har, timings & trace commands ``` _See code: [dist/src/commands/trace.ts](https://github.com/TracerBench/tracerbench/tree/master/packages/cli/blob/v2.3.0/dist/src/commands/trace.ts)_ diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index 15229f08..4f3ff327 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -1 +1,327 @@ -{"version":"2.3.0","commands":{"create-archive":{"id":"create-archive","description":"Creates an automated HAR file from a URL.","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"browserArgs":{"name":"browserArgs","type":"option","description":"(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.","required":true,"default":["--crash-dumps-dir=./tmp","--disable-background-timer-throttling","--disable-dev-shm-usage","--disable-cache","--disable-v8-idle-tasks","--disable-breakpad","--disable-notifications","--disable-hang-monitor","--safebrowsing-disable-auto-update","--ignore-certificate-errors","--v8-cache-options=none"]},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"url":{"name":"url","type":"option","description":"URL to visit for create-archive, timings & trace commands","required":true,"default":"http://localhost:8000/"}},"args":[]},"":{"id":"","description":"Creates an automated HAR file from a URL.","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"browserArgs":{"name":"browserArgs","type":"option","description":"(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.","required":true,"default":["--crash-dumps-dir=./tmp","--disable-background-timer-throttling","--disable-dev-shm-usage","--disable-cache","--disable-v8-idle-tasks","--disable-breakpad","--disable-notifications","--disable-hang-monitor","--safebrowsing-disable-auto-update","--ignore-certificate-errors","--v8-cache-options=none"]},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"url":{"name":"url","type":"option","description":"URL to visit for create-archive, timings & trace commands","required":true,"default":"http://localhost:8000/"}},"args":[]},"marker-timings":{"id":"marker-timings","description":"Get list of all user-timings from trace","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"filter":{"name":"filter","type":"option","description":"User timing marks start with"},"url":{"name":"url","type":"option","description":"URL to visit for create-archive, timings & trace commands","required":true,"default":"http://localhost:8000/"},"traceFrame":{"name":"traceFrame","type":"option","description":"Specify a trace insights frame"}},"args":[]},"report":{"id":"report","description":"Parses the output json from tracerbench and formats it into pdf and html","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."}},"args":[]},"trace":{"id":"trace","description":"Parses a CPU profile and aggregates time across heuristics. Can optinally be vertically sliced with event names.","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"cpuThrottleRate":{"name":"cpuThrottleRate","type":"option","description":"CPU throttle multiplier","required":true,"default":2},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"network":{"name":"network","type":"option","description":"Simulated network conditions.","options":["none | offline | dialup | 2g | edge | slow-3g | em-3g | dsl | 3g | fast-3g | 4g | cable | LTE | FIOS"],"default":"none"},"url":{"name":"url","type":"option","description":"URL to visit for create-archive, timings & trace commands","required":true,"default":"http://localhost:8000/"},"iterations":{"name":"iterations","type":"option","description":"Number of runs","required":true,"default":1},"locations":{"name":"locations","type":"option","description":"include locations in names"},"insights":{"name":"insights","type":"boolean","description":"Analyze insights from command.","allowNo":false}},"args":[]},"compare:analyze":{"id":"compare:analyze","description":"Run an analysis of a benchmark run from a results json file and output to terminal","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. High is the longest trace time.","required":true,"default":"low"},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"}},"args":[{"name":"resultsFile","description":"Results JSON file","required":true}]},"compare":{"id":"compare","description":"Compare the performance delta between an experiment and control","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"hideAnalysis":{"name":"hideAnalysis","type":"boolean","description":"Hide the the analysis output in terminal","allowNo":false},"browserArgs":{"name":"browserArgs","type":"option","description":"(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.","required":true,"default":["--crash-dumps-dir=./tmp","--disable-background-timer-throttling","--disable-dev-shm-usage","--disable-cache","--disable-v8-idle-tasks","--disable-breakpad","--disable-notifications","--disable-hang-monitor","--safebrowsing-disable-auto-update","--ignore-certificate-errors","--v8-cache-options=none"]},"cpuThrottleRate":{"name":"cpuThrottleRate","type":"option","description":"CPU throttle multiplier","required":true,"default":2},"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. High is the longest trace time.","required":true,"default":"low"},"markers":{"name":"markers","type":"option","description":"User Timing Markers","required":true,"default":"domComplete"},"network":{"name":"network","type":"option","description":"Simulated network conditions.","required":true,"options":["none | offline | dialup | 2g | edge | slow-3g | em-3g | dsl | 3g | fast-3g | 4g | cable | LTE | FIOS"],"default":"none"},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"controlURL":{"name":"controlURL","type":"option","description":"Control URL to visit for compare command","required":true,"default":"http://localhost:8000/"},"experimentURL":{"name":"experimentURL","type":"option","description":"Experiment URL to visit for compare command","required":true,"default":"http://localhost:8001/"},"tracingLocationSearch":{"name":"tracingLocationSearch","type":"option","description":"The document location search param.","required":true,"default":"?tracing"},"emulateDevice":{"name":"emulateDevice","type":"option","description":"Emulate a mobile device screen size.","options":["iphone-4","iphone-5se","iphone-678","iphone-678-plus","iphone-x","blackberry-z30","nexus-4","nexus-5","nexus-5x","nexus-6","nexus-6p","pixel-2","pixel-2-xl","lg-optimus-l70","nokia-n9","nokia-lumia-520","microsoft-lumia-550","microsoft-lumia-950","galaxy-s-iii","galaxy-s5","kindle-fire-hdx","ipad-mini","ipad","ipad-pro","blackberry-playbook","nexus-10","nexus-7","galaxy-note-3","galaxy-note-ii","laptop-with-touch","laptop-with-hidpi-screen","laptop-with-mdpi-screen"],"default":""},"emulateDeviceOrientation":{"name":"emulateDeviceOrientation","type":"option","description":"Expected to be either \"vertical\" or \"horizontal\". Dictates orientation of device screen.","options":["horizontal","vertical"],"default":"vertical"},"socksPorts":{"name":"socksPorts","type":"option","description":"Specify a socks proxy port as browser option for control and experiment"},"regressionThreshold":{"name":"regressionThreshold","type":"option","description":"Regression threshold in negative milliseconds. eg -100ms","default":"0ms"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"runtimeStats":{"name":"runtimeStats","type":"boolean","description":"Compare command output deep-dive stats during run.","allowNo":false},"report":{"name":"report","type":"boolean","description":"Generate a PDF report directly after running the compare command.","allowNo":false},"debug":{"name":"debug","type":"boolean","description":"Debug flag per command. Will output noisy command","allowNo":false},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false}},"args":[]}}} \ No newline at end of file +{ + "version": "2.2.4", + "commands": { + "marker-timings": { + "id": "marker-timings", + "description": "Get list of all user-timings from trace", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "tbResultsFolder": { + "name": "tbResultsFolder", + "type": "option", + "description": "The output folder path for all tracerbench results", + "required": true, + "default": "./tracerbench-results" + }, + "filter": { + "name": "filter", + "type": "option", + "description": "User timing marks start with" + }, + "url": { + "name": "url", + "type": "option", + "description": "URL to visit for record-har, timings & trace commands", + "required": true, + "default": "http://localhost:8000/" + }, + "traceFrame": { + "name": "traceFrame", + "type": "option", + "description": "Specify a trace insights frame" + } + }, + "args": [] + }, + "record-har": { + "id": "record-har", + "description": "Generates a HAR file from a URL.", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "url": { + "name": "url", + "type": "option", + "description": "URL to visit for record-har, timings & trace commands", + "required": true + }, + "dest": { + "name": "dest", + "type": "option", + "description": "The destination path for the generated file", + "required": true, + "default": "" + }, + "cookiespath": { + "name": "cookiespath", + "type": "option", + "description": "The path to a JSON file containing cookies to authenticate against the correlated URL", + "required": true + }, + "filename": { + "name": "filename", + "type": "option", + "description": "The filename for the generated file", + "required": true, + "default": "tracerbench" + } + }, + "args": [] + }, + "report": { + "id": "report", + "description": "Parses the output json from tracerbench and formats it into pdf and html", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "tbResultsFolder": { + "name": "tbResultsFolder", + "type": "option", + "description": "The output folder path for all tracerbench results", + "required": true, + "default": "./tracerbench-results" + }, + "config": { + "name": "config", + "type": "option", + "description": "Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all." + } + }, + "args": [] + }, + "trace": { + "id": "trace", + "description": "Parses a CPU profile and aggregates time across heuristics. Can optinally be vertically sliced with event names.", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "cpuThrottleRate": { + "name": "cpuThrottleRate", + "type": "option", + "description": "CPU throttle multiplier", + "required": true, + "default": 2 + }, + "tbResultsFolder": { + "name": "tbResultsFolder", + "type": "option", + "description": "The output folder path for all tracerbench results", + "required": true, + "default": "./tracerbench-results" + }, + "harpath": { + "name": "harpath", + "type": "option", + "description": "The path to the HTTP Archive File (HAR)", + "required": true + }, + "network": { + "name": "network", + "type": "option", + "description": "Simulated network conditions.", + "options": ["none | offline | dialup | 2g | edge | slow-3g | em-3g | dsl | 3g | fast-3g | 4g | cable | LTE | FIOS"], + "default": "none" + }, + "url": { + "name": "url", + "type": "option", + "description": "URL to visit for record-har, timings & trace commands", + "required": true, + "default": "http://localhost:8000/" + }, + "cookiespath": { + "name": "cookiespath", + "type": "option", + "description": "The path to a JSON file containing cookies to authenticate against the correlated URL", + "required": true + }, + "iterations": { + "name": "iterations", + "type": "option", + "description": "Number of runs", + "required": true, + "default": 1 + }, + "locations": { + "name": "locations", + "type": "option", + "description": "include locations in names" + }, + "insights": { + "name": "insights", + "type": "boolean", + "description": "Analyze insights from command.", + "allowNo": false + } + }, + "args": [] + }, + "compare:analyze": { + "id": "compare:analyze", + "description": "Run an analysis of a benchmark run from a results json file and output to terminal", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "fidelity": { + "name": "fidelity", + "type": "option", + "description": "Directly correlates to the number of samples per trace. High is the longest trace time.", + "required": true, + "default": "low" + }, + "tbResultsFolder": { + "name": "tbResultsFolder", + "type": "option", + "description": "The output folder path for all tracerbench results", + "required": true, + "default": "./tracerbench-results" + } + }, + "args": [{ + "name": "resultsFile", + "description": "Results JSON file", + "required": true + }] + }, + "compare": { + "id": "compare", + "description": "Compare the performance delta between an experiment and control", + "pluginName": "tracerbench", + "pluginType": "core", + "aliases": [], + "flags": { + "hideAnalysis": { + "name": "hideAnalysis", + "type": "boolean", + "description": "Hide the the analysis output in terminal", + "allowNo": false + }, + "browserArgs": { + "name": "browserArgs", + "type": "option", + "description": "(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.", + "required": true, + "default": ["--crash-dumps-dir=./tmp", "--disable-background-timer-throttling", "--disable-dev-shm-usage", "--disable-cache", "--disable-v8-idle-tasks", "--disable-breakpad", "--disable-notifications", "--disable-hang-monitor", "--safebrowsing-disable-auto-update", "--ignore-certificate-errors", "--v8-cache-options=none"] + }, + "cpuThrottleRate": { + "name": "cpuThrottleRate", + "type": "option", + "description": "CPU throttle multiplier", + "required": true, + "default": 2 + }, + "fidelity": { + "name": "fidelity", + "type": "option", + "description": "Directly correlates to the number of samples per trace. High is the longest trace time.", + "required": true, + "default": "low" + }, + "markers": { + "name": "markers", + "type": "option", + "description": "User Timing Markers", + "required": true, + "default": "domComplete" + }, + "network": { + "name": "network", + "type": "option", + "description": "Simulated network conditions.", + "required": true, + "options": ["none | offline | dialup | 2g | edge | slow-3g | em-3g | dsl | 3g | fast-3g | 4g | cable | LTE | FIOS"], + "default": "none" + }, + "tbResultsFolder": { + "name": "tbResultsFolder", + "type": "option", + "description": "The output folder path for all tracerbench results", + "required": true, + "default": "./tracerbench-results" + }, + "controlURL": { + "name": "controlURL", + "type": "option", + "description": "Control URL to visit for compare command", + "required": true, + "default": "http://localhost:8000/" + }, + "experimentURL": { + "name": "experimentURL", + "type": "option", + "description": "Experiment URL to visit for compare command", + "required": true, + "default": "http://localhost:8001/" + }, + "tracingLocationSearch": { + "name": "tracingLocationSearch", + "type": "option", + "description": "The document location search param.", + "required": true, + "default": "?tracing" + }, + "emulateDevice": { + "name": "emulateDevice", + "type": "option", + "description": "Emulate a mobile device screen size.", + "options": ["iphone-4", "iphone-5se", "iphone-678", "iphone-678-plus", "iphone-x", "blackberry-z30", "nexus-4", "nexus-5", "nexus-5x", "nexus-6", "nexus-6p", "pixel-2", "pixel-2-xl", "lg-optimus-l70", "nokia-n9", "nokia-lumia-520", "microsoft-lumia-550", "microsoft-lumia-950", "galaxy-s-iii", "galaxy-s5", "kindle-fire-hdx", "ipad-mini", "ipad", "ipad-pro", "blackberry-playbook", "nexus-10", "nexus-7", "galaxy-note-3", "galaxy-note-ii", "laptop-with-touch", "laptop-with-hidpi-screen", "laptop-with-mdpi-screen"], + "default": "" + }, + "emulateDeviceOrientation": { + "name": "emulateDeviceOrientation", + "type": "option", + "description": "Expected to be either \"vertical\" or \"horizontal\". Dictates orientation of device screen.", + "options": ["horizontal", "vertical"], + "default": "vertical" + }, + "socksPorts": { + "name": "socksPorts", + "type": "option", + "description": "Specify a socks proxy port as browser option for control and experiment" + }, + "regressionThreshold": { + "name": "regressionThreshold", + "type": "option", + "description": "Regression threshold in negative milliseconds. eg -100ms", + "default": "0ms" + }, + "config": { + "name": "config", + "type": "option", + "description": "Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all." + }, + "runtimeStats": { + "name": "runtimeStats", + "type": "boolean", + "description": "Compare command output deep-dive stats during run.", + "allowNo": false + }, + "report": { + "name": "report", + "type": "boolean", + "description": "Generate a PDF report directly after running the compare command.", + "allowNo": false + }, + "debug": { + "name": "debug", + "type": "boolean", + "description": "Debug flag per command. Will output noisy command", + "allowNo": false + }, + "headless": { + "name": "headless", + "type": "boolean", + "description": "Run with headless chrome flags", + "allowNo": false + } + }, + "args": [] + } + } +} \ No newline at end of file diff --git a/packages/cli/src/command-config/default-flag-args.ts b/packages/cli/src/command-config/default-flag-args.ts index ef5a203e..7061518d 100644 --- a/packages/cli/src/command-config/default-flag-args.ts +++ b/packages/cli/src/command-config/default-flag-args.ts @@ -63,7 +63,7 @@ export const defaultFlagArgs: ITBConfig = { emulateDevice: '', emulateDeviceOrientation: 'vertical', regressionThreshold: '0ms', - cookie: '', + dest: '', }; // specify with --headless flag diff --git a/packages/cli/src/commands/compare/index.ts b/packages/cli/src/commands/compare/index.ts index b57cc91d..47bc4151 100644 --- a/packages/cli/src/commands/compare/index.ts +++ b/packages/cli/src/commands/compare/index.ts @@ -105,6 +105,7 @@ export default class Compare extends TBBaseCommand { public explicitFlags: string[]; public analyzedJSONString: string = ''; constructor(argv: string[], config: IConfig) { + // config === oclif config NOT tbconfig super(argv, config); const { flags } = this.parse(Compare); this.explicitFlags = argv; diff --git a/packages/cli/src/commands/create-archive.ts b/packages/cli/src/commands/create-archive.ts deleted file mode 100644 index 1f36ba25..00000000 --- a/packages/cli/src/commands/create-archive.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { TBBaseCommand } from '../command-config'; -import { harTrace } from '@tracerbench/core'; -import { browserArgs, tbResultsFolder, url } from '../helpers/flags'; -import * as path from 'path'; -import * as fs from 'fs-extra'; - -export default class CreateArchive extends TBBaseCommand { - public static description = 'Creates an automated HAR file from a URL.'; - public static flags = { - browserArgs: browserArgs({ required: true }), - tbResultsFolder: tbResultsFolder({ required: true }), - url: url({ required: true }), - }; - - public async run() { - const { flags } = this.parse(CreateArchive); - const { browserArgs, url, tbResultsFolder } = flags; - const archiveOutput = path.join(tbResultsFolder, 'trace.har'); - const cookiesJSON = path.join(tbResultsFolder, 'cookies.json'); - let cookies; - let harArchive; - - if (!fs.existsSync(tbResultsFolder)) { - fs.mkdirSync(tbResultsFolder); - } - - [cookies, harArchive] = await harTrace(url, browserArgs); - fs.writeFileSync(cookiesJSON, JSON.stringify(cookies)); - fs.writeFileSync(archiveOutput, JSON.stringify(harArchive)); - - this.log( - `Captured ${harArchive.log.entries.length} request responses in har file.` - ); - return this.log( - `HAR & cookies.json successfully generated from ${url} and available here: ${archiveOutput} and ${cookiesJSON}` - ); - } -} diff --git a/packages/cli/src/commands/marker-timings.ts b/packages/cli/src/commands/marker-timings.ts index a1dd3d17..c250a0e0 100644 --- a/packages/cli/src/commands/marker-timings.ts +++ b/packages/cli/src/commands/marker-timings.ts @@ -13,7 +13,7 @@ import { isFrameNavigationStart, isMark, isUserMark, - loadTraceFile, + setTraceEvents, } from '../helpers/utils'; export default class MarkerTimings extends TBBaseCommand { @@ -51,7 +51,7 @@ export default class MarkerTimings extends TBBaseCommand { } try { - trace = loadTraceFile(rawTraceData); + trace = setTraceEvents(rawTraceData); } catch (error) { this.error(`${error}`); } diff --git a/packages/cli/src/commands/record-har.ts b/packages/cli/src/commands/record-har.ts new file mode 100644 index 00000000..ffbf6f62 --- /dev/null +++ b/packages/cli/src/commands/record-har.ts @@ -0,0 +1,101 @@ +import { Command } from '@oclif/command'; +import { recordHARClient } from '@tracerbench/core'; +import { readJson, writeFileSync } from 'fs-extra'; +import * as path from 'path'; +import * as tmp from 'tmp'; + +import { dest, url, cookiespath, filename } from '../helpers/flags'; + +tmp.setGracefulCleanup(); + +export default class RecordHAR extends Command { + public static description = 'Generates a HAR file from a URL.'; + public static flags = { + url: url({ required: true, default: undefined }), + dest: dest({ required: true }), + cookiespath: cookiespath({ required: true }), + filename: filename({ required: true, default: 'tracerbench' }), + }; + + public async run() { + const { flags } = this.parse(RecordHAR); + const { url, dest, cookiespath, filename } = flags; + + // grab the auth cookies + const cookies = await readJson(path.resolve(cookiespath)); + + // record the actual HAR and return the archive file + const harArchive = await recordHARClient(url, getBrowserArgs(), cookies); + + writeFileSync( + path.join(dest, `${filename}.har`), + JSON.stringify(harArchive) + ); + + this.log( + `HAR recorded and available here: ${path.join(dest, filename)}.har` + ); + } +} + +function getBrowserArgs(): string[] { + interface IViewOptions { + windowSize: { + width: number; + height: number; + }; + deviceScaleFactor: number; + userAgent: string | undefined; + } + + const tmpDir = tmp.dirSync({ + unsafeCleanup: true, + }); + + const options: IViewOptions = { + windowSize: { + width: 1280, + height: 800, + }, + deviceScaleFactor: 0, + userAgent: undefined, + }; + + return [ + `--crash-dumps-dir=${tmpDir.name}`, + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-component-extensions-with-background-pages', + '--disable-client-side-phishing-detection', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-domain-reliability', + '--disable-extensions', + '--disable-features=NetworkPrediction', + '--disable-features=site-per-process,TranslateUI,BlinkGenPropertyTrees', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-notifications', + '--disable-renderer-backgrounding', + '--disable-sync', + '--disable-translate', + '--disable-v8-idle-tasks', + `--device-scale-factor=${options.deviceScaleFactor}`, + '--ignore-certificate-errors-spki-list=uU0W87bsSHNaY+g/o8S9PmyxIgf92JepLWrPg5bYb+s=', + '--metrics-recording-only', + '--no-pings', + '--no-first-run', + '--no-default-browser-check', + '--no-experiments', + '--no-sandbox', + '--password-store=basic', + '--safebrowsing-disable-auto-update', + '--use-mock-keychain', + `--user-agent=${options.userAgent}`, + `--user-data-dir=${tmpDir.name}`, + '--v8-cache-options=none', + `--window-size=${options.windowSize.width},${options.windowSize.height}`, + '--headless', + ]; +} diff --git a/packages/cli/src/commands/trace.ts b/packages/cli/src/commands/trace.ts index 1209ee95..86ce892a 100644 --- a/packages/cli/src/commands/trace.ts +++ b/packages/cli/src/commands/trace.ts @@ -1,12 +1,13 @@ import { TBBaseCommand } from '../command-config'; -import * as fs from 'fs-extra'; +import { readJson } from 'fs-extra'; import * as path from 'path'; import { liveTrace, - harTrace, analyze, loadTrace, ITraceEvent, + IConditions, + IAnalyze, } from '@tracerbench/core'; import { tbResultsFolder, @@ -16,12 +17,13 @@ import { url, insights, locations, + harpath, + cookiespath, } from '../helpers/flags'; import { - getCookiesFromHAR, normalizeFnName, isCommitLoad, - loadTraceFile, + setTraceEvents, } from '../helpers/utils'; export default class Trace extends TBBaseCommand { @@ -29,8 +31,10 @@ export default class Trace extends TBBaseCommand { public static flags = { cpuThrottleRate: cpuThrottleRate({ required: true }), tbResultsFolder: tbResultsFolder({ required: true }), + harpath: harpath({ required: true }), network: network(), url: url({ required: true }), + cookiespath: cookiespath({ required: true }), iterations: iterations({ required: true }), locations: locations(), insights, @@ -41,76 +45,36 @@ export default class Trace extends TBBaseCommand { const { url, cpuThrottleRate, + cookiespath, tbResultsFolder, insights, locations, + network, + harpath, } = flags; - const network = 'none'; - const cpu = cpuThrottleRate; - const file = tbResultsFolder; - const event = undefined; - const report = undefined; - const methods = ['']; - const traceJSON = path.join(tbResultsFolder, 'trace.json'); - const traceHAR = path.join(tbResultsFolder, 'trace.har'); - const cookiesJSON = path.join(tbResultsFolder, 'cookies.json'); - - let archiveFile; - let rawTraceData; - let cookies: any = ''; - - // if trace can't find a HAR then go and record one - if (!fs.existsSync(traceHAR)) { - [cookies, archiveFile] = await harTrace(url); - - fs.writeFileSync(cookiesJSON, JSON.stringify(cookies)); - fs.writeFileSync(traceHAR, JSON.stringify(archiveFile)); - } - try { - cookies = JSON.parse(fs.readFileSync(cookiesJSON, 'utf8')); - } catch (error) { - try { - cookies = getCookiesFromHAR( - JSON.parse(fs.readFileSync(traceHAR, 'utf8')) - ); - } catch (error) { - this.error( - `Could not extract cookies from cookies.json or HAR file at path ${traceHAR}, ${error}` - ); - cookies = null; - } - } + const methods = ['']; - await liveTrace(url, traceJSON, cookies, { - cpu, + const traceJSONFile = path.resolve(tbResultsFolder, 'trace.json'); + const traceJSON = await readJson(traceJSONFile); + const cookiesJSON = await readJson(path.resolve(cookiespath)); + const traceHAR = path.resolve(harpath); + const traceHARJSON = await readJson(traceHAR); + const conditions: IConditions = { + cpu: cpuThrottleRate, network, - }); - - try { - rawTraceData = JSON.parse(fs.readFileSync(traceJSON, 'utf8')); - } catch (error) { - this.error( - `Could not extract trace events from '${traceJSON}', ${error}` - ); - } + }; + const analyzeOptions: IAnalyze = { + traceJSON, + traceHARJSON, + methods, + }; - try { - archiveFile = JSON.parse(fs.readFileSync(traceHAR, 'utf8')); - } catch (error) { - this.error( - `Could not find trace har file at path: ${traceHAR}, ${error}` - ); - } + // run the liveTrace + await liveTrace(url, traceJSONFile, cookiesJSON, conditions); - await analyze({ - archiveFile, - event, - file, - methods, - rawTraceData, - report, - }); + // analyze the liveTrace + await analyze(analyzeOptions); if (insights) { // js-eval-time @@ -121,7 +85,7 @@ export default class Trace extends TBBaseCommand { const methods = new Set(); try { - trace = loadTraceFile(rawTraceData); + trace = setTraceEvents(traceJSON); } catch (error) { this.error(`${error}`); } @@ -184,13 +148,8 @@ export default class Trace extends TBBaseCommand { this.error(error); } - // // list-functions - // methods.forEach(method => - // this.log(`Successfully listed method: ${method}`) - // ); - try { - trace = loadTraceFile(rawTraceData); + trace = setTraceEvents(traceJSON); const traceLoad = trace.filter(isCommitLoad); traceLoad.forEach( ({ diff --git a/packages/cli/src/helpers/flags.ts b/packages/cli/src/helpers/flags.ts index 20d817fa..05cdabfc 100644 --- a/packages/cli/src/helpers/flags.ts +++ b/packages/cli/src/helpers/flags.ts @@ -1,5 +1,4 @@ /* tslint:disable:no-console*/ - import { flags } from '@oclif/command'; import { networkConditions } from '@tracerbench/core'; import Protocol from 'devtools-protocol'; @@ -155,7 +154,7 @@ export const tbResultsFolder = flags.build({ export const url = flags.build({ default: () => getDefaultValue('url'), - description: 'URL to visit for create-archive, timings & trace commands', + description: 'URL to visit for record-har, timings & trace commands', }); export const controlURL = flags.build({ @@ -212,3 +211,24 @@ export const emulateDeviceOrientation = flags.build({ description: `Expected to be either "vertical" or "horizontal". Dictates orientation of device screen.`, options: ['horizontal', 'vertical'], }); + +export const cookiespath = flags.build({ + description: `The path to a JSON file containing cookies to authenticate against the correlated URL`, +}); + +export const tbconfigpath = flags.build({ + description: `The path to a TracerBench configuration file (tbconfig.json)`, +}); + +export const harpath = flags.build({ + description: `The path to the HTTP Archive File (HAR)`, +}); + +export const dest = flags.build({ + default: () => getDefaultValue('dest'), + description: `The destination path for the generated file`, +}); + +export const filename = flags.build({ + description: `The filename for the generated file`, +}); diff --git a/packages/cli/src/helpers/utils.ts b/packages/cli/src/helpers/utils.ts index 0d661ecb..140f8078 100644 --- a/packages/cli/src/helpers/utils.ts +++ b/packages/cli/src/helpers/utils.ts @@ -54,7 +54,8 @@ export function mergeLeft( Object.keys(right).forEach(key => { const leftValue = left[key]; const rightValue = left[key]; - const matchingObjectType = typeof leftValue === 'object' && typeof rightValue === 'object'; + const matchingObjectType = + typeof leftValue === 'object' && typeof rightValue === 'object'; const isOneArray = Array.isArray(leftValue) || Array.isArray(rightValue); if (matchingObjectType && (left[key] || right[key]) && !isOneArray) { @@ -94,7 +95,9 @@ export function normalizeFnName(name: string) { return name; } -export function loadTraceFile(file: any) { +export function setTraceEvents( + file: ITraceEvent[] | { metadata: {}; traceEvents: ITraceEvent[] } +) { if (!Array.isArray(file)) { file = file.traceEvents; } diff --git a/packages/cli/test/commands/record-har.ts b/packages/cli/test/commands/record-har.ts new file mode 100644 index 00000000..51086199 --- /dev/null +++ b/packages/cli/test/commands/record-har.ts @@ -0,0 +1,30 @@ +import { test } from '@oclif/test'; +import * as chai from 'chai'; +import RecordHAR from '../../src/commands/record-har'; +import { COOKIES, TB_RESULTS_FOLDER, URL } from '../test-helpers'; + +chai.use(require('chai-fs')); + +const FILENAME = 'foo'; + +describe('record-har', () => { + test + .stdout() + .it( + `runs record-har --url ${URL} --dest ${TB_RESULTS_FOLDER} --cookiespath ${COOKIES} --filename ${FILENAME}`, + async ctx => { + await RecordHAR.run([ + '--url', + URL, + '--dest', + TB_RESULTS_FOLDER, + '--cookiespath', + COOKIES, + '--filename', + FILENAME, + ]); + chai.expect(ctx.stdout).to.contain(`HAR recorded and available here:`); + chai.expect(`${TB_RESULTS_FOLDER}/${FILENAME}.har`).to.be.a.file(); + } + ); +}); diff --git a/packages/cli/test/commands/report.test.ts b/packages/cli/test/commands/report.test.ts index 4509add5..afd1eace 100644 --- a/packages/cli/test/commands/report.test.ts +++ b/packages/cli/test/commands/report.test.ts @@ -4,8 +4,8 @@ import Compare from '../../src/commands/compare'; import Report from '../../src/commands/report'; import { FIXTURE_APP, - TB_RESULTS_FOLDER, TB_CONFIG_FILE, + TB_RESULTS_FOLDER, } from '../test-helpers'; const fidelity = 'test'; diff --git a/packages/cli/test/commands/trace.ts b/packages/cli/test/commands/trace.ts index 1bd532e3..6f6365ea 100644 --- a/packages/cli/test/commands/trace.ts +++ b/packages/cli/test/commands/trace.ts @@ -1,27 +1,33 @@ import { test } from '@oclif/test'; -import { assert, expect } from 'chai'; +import { use, expect } from 'chai'; import Trace from '../../src/commands/trace'; -import { TB_RESULTS_FOLDER, URL } from '../test-helpers'; +import { COOKIES, TB_RESULTS_FOLDER, URL, HAR_PATH } from '../test-helpers'; -const cpuThrottleRate = '1'; +const chaiFiles = require('chai-files'); +use(chaiFiles); + +const file = chaiFiles.file; describe('trace', () => { test .stdout() .it( - `runs trace --url ${URL} --tbResultsFolder ${TB_RESULTS_FOLDER} --cpuThrottleRate ${cpuThrottleRate}`, + `runs trace --url ${URL} --tbResultsFolder ${TB_RESULTS_FOLDER} --harpath ${HAR_PATH} --cookiespath ${COOKIES}`, async ctx => { await Trace.run([ '--url', URL, '--tbResultsFolder', TB_RESULTS_FOLDER, - '--cpuThrottleRate', - cpuThrottleRate, + '--harpath', + HAR_PATH, + '--cookiespath', + COOKIES, ]); expect(ctx.stdout).to.contain(`Trace`); expect(ctx.stdout).to.contain(`Subtotal`); - assert.exists(`${TB_RESULTS_FOLDER}/trace.json`); + // tslint:disable-next-line: no-unused-expression + expect(file(`${TB_RESULTS_FOLDER}/trace.json`)).to.exist; } ); }); @@ -30,15 +36,17 @@ describe('trace: insights', () => { test .stdout() .it( - `runs trace --url ${URL} --tbResultsFolder ${TB_RESULTS_FOLDER} --cpuThrottleRate ${cpuThrottleRate} --insights`, + `runs trace --url ${URL} --tbResultsFolder ${TB_RESULTS_FOLDER} --harpath ${HAR_PATH} --cookiespath ${COOKIES} --insights`, async ctx => { await Trace.run([ '--url', URL, '--tbResultsFolder', TB_RESULTS_FOLDER, - '--cpuThrottleRate', - cpuThrottleRate, + '--harpath', + HAR_PATH, + '--cookiespath', + COOKIES, '--insights', ]); expect(ctx.stdout).to.contain(`.js`); diff --git a/packages/cli/test/fixtures/fixture.har b/packages/cli/test/fixtures/fixture.har new file mode 100644 index 00000000..5ed254ab --- /dev/null +++ b/packages/cli/test/fixtures/fixture.har @@ -0,0 +1,36 @@ +{ + "log": { + "entries": [ + { + "request": { + "url": "https://www.tracerbench.com/" + }, + "response": { + "content": { + "text": "foo-1" + } + } + }, + { + "request": { + "url": "https://www.tracerbench.com/assets/tracerbench-663143c6834a3b980de194d173f205dc.js" + }, + "response": { + "content": { + "text": "foo-2" + } + } + }, + { + "request": { + "url": "https://www.tracerbench.com/assets/vendor-0be76fcbd5273c6f91fa37bdd48b6d3c.js" + }, + "response": { + "content": { + "text": "foo-3" + } + } + } + ] + } +} diff --git a/packages/cli/test/mocha.opts b/packages/cli/test/mocha.opts index 7a6e97e9..f1d5b30a 100644 --- a/packages/cli/test/mocha.opts +++ b/packages/cli/test/mocha.opts @@ -5,5 +5,5 @@ --reporter spec --timeout 60000 --file ./test/setup.ts ---file ./test/commands/create-archive.ts +--file ./test/commands/record-har.ts --file ./test/commands/trace.ts \ No newline at end of file diff --git a/packages/cli/test/test-helpers.ts b/packages/cli/test/test-helpers.ts index 2c9cb01e..1e6253ce 100644 --- a/packages/cli/test/test-helpers.ts +++ b/packages/cli/test/test-helpers.ts @@ -1,4 +1,5 @@ import { pathToFileURL } from 'url'; +import { writeFileSync, mkdirpSync } from 'fs-extra'; import { tmpDir } from './setup'; import { join, resolve } from 'path'; @@ -23,4 +24,34 @@ export const HAR_PATH = resolve( join(process.cwd(), '/test/fixtures/fixture.har') ); export const URL = 'https://www.tracerbench.com'; -export const TRACE = resolve(join(process.cwd(), '/test/fixtures/trace.json')); + +export interface FileStructure { + [key: string]: string | FileStructure; +} + +/** + * Recursively build the file structure according to the tree passed + * + * @param structure - File structure to generate. Any values that are non strings are considered another folder + * @param rootFolder - Generate the file structure in this folder + */ +export function generateFileStructure( + structure: FileStructure, + rootFolder: string +) { + const names = Object.keys(structure); + + names.forEach((name: string) => { + const pathForName = join(rootFolder, name); + if (typeof structure[name] === 'string') { + writeFileSync(pathForName, structure[name]); + } else if ( + typeof structure[name] === 'object' && + Object.keys(structure[name]).length + ) { + mkdirpSync(pathForName); + // @ts-ignore + generateFileStructure(structure[name], pathForName); + } + }); +} diff --git a/packages/tracerbench/src/index.ts b/packages/tracerbench/src/index.ts index 4f57a5b0..27395a3b 100644 --- a/packages/tracerbench/src/index.ts +++ b/packages/tracerbench/src/index.ts @@ -23,10 +23,12 @@ export { Runner, IBenchmark } from './runner'; export { ITab } from './tab'; export { analyze, - harTrace, + recordHARClient, loadTrace, liveTrace, networkConditions, ITraceEvent, + IAnalyze, + IConditions, } from './trace'; export * from './util'; diff --git a/packages/tracerbench/src/trace/analyze.ts b/packages/tracerbench/src/trace/analyze.ts index 586a8e9c..f45e3845 100644 --- a/packages/tracerbench/src/trace/analyze.ts +++ b/packages/tracerbench/src/trace/analyze.ts @@ -1,7 +1,7 @@ -import Trace from './trace'; +import { HierarchyNode } from 'd3-hierarchy'; +import { writeFileSync } from 'fs'; + import { ITrace, loadTrace } from './load_trace'; -import { ITraceEvent } from './trace_event'; -import { exportHierarchy } from './export-hierarchy'; import { aggregate, categorizeAggregations, @@ -9,7 +9,6 @@ import { verifyMethods, } from './aggregator'; import { IArchive } from './archive_trace'; -import { ModuleMatcher } from './module_matcher'; import { report as reporter } from './reporter'; import { addRemainingModules, @@ -18,25 +17,32 @@ import { getRenderingNodes, methodsFromCategories, } from './utils'; +import { ModuleMatcher } from './module_matcher'; +import { + ICpuProfileNode, + ITraceEvent, + Trace, + TRACE_EVENT_PHASE_COMPLETE, +} from '../trace'; -interface IAnalyze { - rawTraceData: ITraceEvent[] | ITrace; - archiveFile: IArchive; +export interface IAnalyze { + traceJSON: ITraceEvent[] | ITrace; + traceHARJSON: IArchive; methods: string[]; + filename?: string; event?: string; report?: string; verbose?: boolean; - file: string; } export async function analyze(options: IAnalyze) { - const { archiveFile, event, file, rawTraceData, report, methods } = options; - const trace = loadTrace(rawTraceData); + const { traceHARJSON, event, filename, traceJSON, report, methods } = options; + const trace = loadTrace(traceJSON); const profile = getCPUProfile(trace, event)!; const { hierarchy } = profile; - const modMatcher = new ModuleMatcher(hierarchy, archiveFile); - exportHierarchy(rawTraceData, hierarchy, trace, file, modMatcher); + const modMatcher = new ModuleMatcher(hierarchy, traceHARJSON); + exportHierarchy(traceJSON, hierarchy, trace, filename, modMatcher); const categories = formatCategories(report, methods); const allMethods = methodsFromCategories(categories); @@ -47,7 +53,7 @@ export async function analyze(options: IAnalyze) { const aggregations = aggregate( hierarchy, allMethods, - archiveFile, + traceHARJSON, modMatcher ); const collapsed = collapseCallFrames(aggregations); @@ -57,7 +63,7 @@ export async function analyze(options: IAnalyze) { const renderNodes = getRenderingNodes(hierarchy); renderNodes.forEach(node => { - const renderAgg = aggregate(node, allMethods, archiveFile, modMatcher); + const renderAgg = aggregate(node, allMethods, traceHARJSON, modMatcher); const renderCollapsed = collapseCallFrames(renderAgg); const renderCategorized = categorizeAggregations( renderCollapsed, @@ -68,6 +74,41 @@ export async function analyze(options: IAnalyze) { }); } +export function exportHierarchy( + rawTraceData: any, + hierarchy: HierarchyNode, + trace: Trace, + filename: string = 'trace-processed', + modMatcher: ModuleMatcher +) { + const newTraceData = JSON.parse(JSON.stringify(rawTraceData)); + hierarchy.each(node => { + const completeEvent: ITraceEvent = { + pid: trace.mainProcess!.id, + tid: trace.mainProcess!.mainThread!.id, + ts: node.data.min, + ph: TRACE_EVENT_PHASE_COMPLETE, + cat: 'blink.user_timing', + name: node.data.callFrame.functionName, + args: { + data: { + functionName: node.data.callFrame.functionName, + moduleName: modMatcher.findModuleName(node.data.callFrame), + }, + }, + dur: node.data.max - node.data.min, + }; + + newTraceData.traceEvents.push(completeEvent); + }); + + writeFileSync( + `${filename}.json`, + JSON.stringify(newTraceData, null, ' '), + 'utf8' + ); +} + function getCPUProfile(trace: Trace, event?: string) { const { min, max } = computeMinMax(trace, 'navigationStart', event!); return trace.cpuProfile(min, max); diff --git a/packages/tracerbench/src/trace/archive_trace.ts b/packages/tracerbench/src/trace/archive_trace.ts index 4f8c0ce5..0c5de1d2 100644 --- a/packages/tracerbench/src/trace/archive_trace.ts +++ b/packages/tracerbench/src/trace/archive_trace.ts @@ -29,26 +29,17 @@ export interface IEntry { response: IResponse; } -export async function harTrace( +export async function recordHARClient( url: string, - additionalBrowserArgs: string[] = [], - cookies: any = null -) { - // the saving of the cookies should be a dif command - // spawn browser > sign-in > done > save cookies - - // passing in the cookies file needs to be more - // explicit (especially as its pertained to automation) - - // in the instance we are passing in the cookies - - const browser = await createBrowser(additionalBrowserArgs); + browserArgs: string[], + cookies: Protocol.Network.CookieParam[] +): Promise { + const browser = await createBrowser(browserArgs); try { const client = await getTab(browser.connection); const requestIds: string[] = []; const responses: Protocol.Network.Response[] = []; - const urls = [url]; client.on('Network.responseReceived', ({ requestId, response }) => { if ( @@ -69,9 +60,6 @@ export async function harTrace( await client.send('Network.enable'); - cookies = cookies - ? cookies - : await client.send('Network.getCookies', { urls }); await setCookies(client, cookies); await client.send('Page.enable'); @@ -93,7 +81,7 @@ export async function harTrace( }; archive.log.entries.push(entry); } - return [cookies, archive]; + return archive; } finally { await browser.dispose(); } diff --git a/packages/tracerbench/src/trace/export-hierarchy.ts b/packages/tracerbench/src/trace/export-hierarchy.ts deleted file mode 100644 index 3f885d76..00000000 --- a/packages/tracerbench/src/trace/export-hierarchy.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { HierarchyNode } from 'd3-hierarchy'; -import * as fs from 'fs'; -import { ModuleMatcher } from './module_matcher'; -import { - ICpuProfileNode, - ITraceEvent, - Trace, - TRACE_EVENT_PHASE_COMPLETE -} from '../trace'; - -export function exportHierarchy( - rawTraceData: any, - hierarchy: HierarchyNode, - trace: Trace, - filePath: string, - modMatcher: ModuleMatcher -) { - const newTraceData = JSON.parse(JSON.stringify(rawTraceData)); - hierarchy.each(node => { - const completeEvent: ITraceEvent = { - pid: trace.mainProcess!.id, - tid: trace.mainProcess!.mainThread!.id, - ts: node.data.min, - ph: TRACE_EVENT_PHASE_COMPLETE, - cat: 'blink.user_timing', - name: node.data.callFrame.functionName, - args: { - data: { - functionName: node.data.callFrame.functionName, - moduleName: modMatcher.findModuleName(node.data.callFrame) - } - }, - dur: node.data.max - node.data.min - }; - - newTraceData.traceEvents.push(completeEvent); - }); - - const outputFilePath = filePath.endsWith('.json') - ? filePath.slice(0, filePath.length - 5) - : filePath; - fs.writeFileSync( - `${outputFilePath}-processed.json`, - JSON.stringify(newTraceData, null, ' '), - 'utf8' - ); -} diff --git a/packages/tracerbench/src/trace/index.ts b/packages/tracerbench/src/trace/index.ts index 8fd8bdb4..0d556c1d 100644 --- a/packages/tracerbench/src/trace/index.ts +++ b/packages/tracerbench/src/trace/index.ts @@ -2,9 +2,9 @@ export { default as Bounds } from './bounds'; export { default as Process } from './process'; export { default as Trace } from './trace'; export { default as Thread } from './thread'; -export { analyze } from './analyze'; -export { harTrace } from './archive_trace'; +export { analyze, IAnalyze } from './analyze'; +export { recordHARClient } from './archive_trace'; export { loadTrace } from './load_trace'; export { liveTrace } from './live_trace'; -export { networkConditions } from './conditions'; +export { networkConditions, IConditions } from './conditions'; export * from './trace_event'; diff --git a/packages/tracerbench/src/trace/live_trace.ts b/packages/tracerbench/src/trace/live_trace.ts index da49edab..1c30b01d 100644 --- a/packages/tracerbench/src/trace/live_trace.ts +++ b/packages/tracerbench/src/trace/live_trace.ts @@ -31,7 +31,7 @@ export async function liveTrace( cookies: Protocol.Network.CookieParam[], conditions: IConditions ) { - const browser = await createBrowser(); + const browser = await createBrowser([`--crash-dumps-dir=/tmp`]); try { const client = await getTab(browser.connection); await emulate(client, conditions); @@ -72,12 +72,12 @@ export async function liveTrace( }); await Promise.all([ client.until('Page.loadEventFired'), - client.send('Page.navigate', { url }) + client.send('Page.navigate', { url }), ]); const [result] = await Promise.all([ client.until('Tracing.tracingComplete'), - client.send('Tracing.end') + client.send('Tracing.end'), ]); const handle = result.stream as string; diff --git a/packages/tracerbench/src/trace/trace-utils.ts b/packages/tracerbench/src/trace/trace-utils.ts index fa7a21b8..9925071a 100644 --- a/packages/tracerbench/src/trace/trace-utils.ts +++ b/packages/tracerbench/src/trace/trace-utils.ts @@ -5,13 +5,9 @@ import Protocol from 'devtools-protocol'; import { IConditions, networkConditions } from './conditions'; import { filterObjectByKeys } from './utils'; -const DEFAULT_BROWSER_ARGS = ['--crash-dumps-dir=/tmp']; - -export async function createBrowser(additionalBrowserArgs: string[] = []) { - const mergedBrowserArgs = additionalBrowserArgs.concat(DEFAULT_BROWSER_ARGS); - +export async function createBrowser(browserArgs: string[] = []) { const browser = await spawnChrome({ - additionalArguments: mergedBrowserArgs, + additionalArguments: browserArgs, stdio: 'inherit', chromeExecutable: undefined, userDataDir: undefined,