Refactor benchmark test suite and add style benchmark tests

mapbox · Aug 27, 2018 · b3eb7da · b3eb7da
1 parent 53e622b
commit b3eb7da
Show file tree

Hide file tree

Showing 28 changed files with 563 additions and 221 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,8 +11,10 @@ coverage
 flow-coverage
 .DS_Store
 .nyc_output
-bench/benchmarks_generated.js
-bench/benchmarks_view_generated.js
+bench/versions/benchmarks_generated.js
+bench/versions/benchmarks_view_generated.js
+bench/styles/benchmarks_generated.js
+bench/styles/benchmarks_view_generated.js
 debug/access_token_generated.js
 debug/style_property_editor_generated.js
 test/integration/**/index*.html

diff --git a/bench/README.md b/bench/README.md
@@ -10,12 +10,26 @@ Start the benchmark server
 MAPBOX_ACCESS_TOKEN={YOUR MAPBOX ACCESS TOKEN} yarn start
 ```
 
-To run all benchmarks, open [the benchmark page, `http://localhost:9966/bench`](http://localhost:9966/bench).
+To run all benchmarks, open [the benchmark page, `http://localhost:9966/bench/versions`](http://localhost:9966/bench/versions).
 
-To run a specific benchmark, add its name to the url hash, for example [`http://localhost:9966/bench/#Layout`](http://localhost:9966/bench/#Layout).
+To run a specific benchmark, add its name to the url hash, for example [`http://localhost:9966/bench/versions#Layout`](http://localhost:9966/bench/versions#Layout).
 
-By default, the benchmark page will compare the local branch against `master` and the latest release. To change this, include one or more `compare` query parameters in the URL: E.g., [localhost:9966/bench?compare=master](http://localhost:9966/bench?compare=master) or [localhost:9966/bench?compare=master#Layout](http://localhost:9966/bench?compare=master#Layout) to compare only to master, or [localhost:9966/bench?compare=v0.44.0&compare=v0.44.1](http://localhost:9966/bench?compare=v0.44.0&compare=v0.44.1) to compare to `v0.44.0` and `v0.44.1` (but not `master`).  Versions available for comparison are: `master` and `vX.Y.Z` for versions >= `v0.41.0`.
+By default, the benchmark page will compare the local branch against `master` and the latest release. To change this, include one or more `compare` query parameters in the URL: E.g., [localhost:9966/bench/versions?compare=master](http://localhost:9966/bench/versions?compare=master) or [localhost:9966/bench/versions?compare=master#Layout](http://localhost:9966/bench/versions?compare=master#Layout) to compare only to master, or [localhost:9966/bench/versions?compare=v0.44.0&compare=v0.44.1](http://localhost:9966/bench/versions?compare=v0.44.0&compare=v0.44.1) to compare to `v0.44.0` and `v0.44.1` (but not `master`).  Versions available for comparison are: `master` and `vX.Y.Z` for versions >= `v0.41.0`.
 
+## Running Style Benchmarks
+
+Start the benchmark server
+
+```bash
+MAPBOX_STYLE_URL={YOUR STYLES HERE} yarn start
+```
+Note: `MAPBOX_STYLE_URL` takes a comma-separated list of up to 3 Mapbox style URLs (e.g. `mapbox://styles/mapbox/streets-v10,mapbox://styles/mapbox/streets-v9`)
+
+To run all benchmarks, open [the benchmark page, `http://localhost:9966/bench/styles`](http://localhost:9966/bench/styles).
+
+To run a specific benchmark, add its name to the url hash, for example [`http://localhost:9966/bench/styles#Layout`](http://localhost:9966/bench/styles#Layout).
+
+By default, the style benchmark page will run its benchmarks against `mapbox://styles/mapbox/streets-v10`. `Layout` and `Paint` styles will run one instance of the test for each tile/location in an internal list of tiles. This behavior helps visualize the ways in which a style performs given various conditions present in each tile (CJK text, dense urban areas, rural areas, etc). `QueryBox` and `QueryPoint` use the internal list of tiles but otherwise run the same as their non-style benchmark equivalents. `StyleLayerCreate` and `StyleValidate` are not tile/location dependent and run the same way as their non-style benchmark equivalents. All other benchmark tests from the non-style suite are not used when benchmarking styles.
 
 ## Writing a Benchmark
 

diff --git a/bench/benchmarks.js b/bench/benchmarks.js
diff --git a/bench/benchmarks/expressions.js b/bench/benchmarks/expressions.js
@@ -2,11 +2,11 @@
 
 import Benchmark from '../lib/benchmark';
 
-import accessToken from '../lib/access_token';
 import spec from '../../src/style-spec/reference/latest';
 import convertFunction from '../../src/style-spec/function/convert';
 import { isFunction, createFunction } from '../../src/style-spec/function';
 import { createPropertyExpression } from '../../src/style-spec/expression';
+import { normalizeStyleURL } from '../../src/util/mapbox';
 
 import type {StylePropertySpecification} from '../../src/style-spec/style-spec';
 import type {StylePropertyExpression} from '../../src/style-spec/expression';
@@ -19,9 +19,15 @@ class ExpressionBenchmark extends Benchmark {
         compiledFunction: StylePropertyExpression,
         compiledExpression: StylePropertyExpression
     }>;
+    style: string;
+
+    constructor(style: string) {
+        super();
+        this.style = style;
+    }
 
     setup() {
-        return fetch(`https://api.mapbox.com/styles/v1/mapbox/streets-v9?access_token=${accessToken}`)
+        return fetch(normalizeStyleURL(this.style))
             .then(response => response.json())
             .then(json => {
                 this.data = [];

diff --git a/bench/benchmarks/layers.js b/bench/benchmarks/layers.js
@@ -3,6 +3,9 @@ import Benchmark from '../lib/benchmark';
 import createMap from '../lib/create_map';
 import style from '../data/empty.json';
 
+const width = 1024;
+const height = 768;
+
 function generateLayers(layer) {
     const generated = [];
     for (let i = 0; i < 50; i++) {
@@ -16,8 +19,8 @@ class LayerBenchmark extends Benchmark {
     setup() {
         return createMap({
             zoom: 16,
-            width: 1024,
-            height: 768,
+            width,
+            height,
             center: [-77.032194, 38.912753],
             style: this.layerStyle
         }).then(map => {

diff --git a/bench/benchmarks/layout.js b/bench/benchmarks/layout.js
@@ -8,21 +8,32 @@ import { OverscaledTileID } from '../../src/source/tile_id';
 export default class Layout extends Benchmark {
     tiles: Array<{tileID: OverscaledTileID, buffer: ArrayBuffer}>;
     parser: TileParser;
+    style: string;
+    locations: Array<OverscaledTileID>;
 
-    setup(): Promise<void> {
-        const tileIDs = [
+    constructor(style: string, locations: ?Array<OverscaledTileID>) {
+        super();
+        this.style = style;
+        this.locations = locations || this.tileIDs();
+    }
+
+    tileIDs(): Array<OverscaledTileID> {
+        return [
             new OverscaledTileID(12, 0, 12, 655, 1583),
             new OverscaledTileID(8, 0, 8, 40, 98),
             new OverscaledTileID(4, 0, 4, 3, 6),
             new OverscaledTileID(0, 0, 0, 0, 0)
         ];
-        return fetchStyle(`mapbox://styles/mapbox/streets-v9`)
+    }
+
+    setup(): Promise<void> {
+        return fetchStyle(this.style)
             .then((styleJSON) => {
                 this.parser = new TileParser(styleJSON, 'composite');
                 return this.parser.setup();
             })
             .then(() => {
-                return Promise.all(tileIDs.map(tileID => this.parser.fetchTile(tileID)));
+                return Promise.all(this.locations.map(tileID => this.parser.fetchTile(tileID)));
             })
             .then((tiles) => {
                 this.tiles = tiles;

diff --git a/bench/benchmarks/paint.js b/bench/benchmarks/paint.js
@@ -1,24 +1,36 @@
+// @flow
 
 import Benchmark from '../lib/benchmark';
 import createMap from '../lib/create_map';
+import type Map from '../../src/ui/map';
 
 const width = 1024;
 const height = 768;
-const zooms = [4, 8, 11, 13, 15, 17];
 
 export default class Paint extends Benchmark {
-    setup() {
-        return Promise.all(zooms.map(zoom => {
+    style: string;
+    locations: Array<Object>;
+    maps: Array<Map>;
+
+    constructor(style: string, locations: Array<Object>) {
+        super();
+        this.style = style;
+        this.locations = locations;
+    }
+
+    setup(): Promise<void> {
+        return Promise.all(this.locations.map(location => {
             return createMap({
-                zoom,
+                zoom: location.zoom,
                 width,
                 height,
-                center: [-77.032194, 38.912753],
-                style: 'mapbox://styles/mapbox/streets-v9'
+                center: location.center,
+                style: this.style
+            });
+        }))
+            .then(maps => {
+                this.maps = maps;
             });
-        })).then(maps => {
-            this.maps = maps;
-        });
     }
 
     bench() {

diff --git a/bench/benchmarks/paint_states.js b/bench/benchmarks/paint_states.js
@@ -17,6 +17,11 @@ const height = 768;
 const zoom = 4;
 
 export default class PaintStates extends Benchmark {
+    constructor(center) {
+        super();
+        this.center = center;
+    }
+
     setup() {
         return fetch('/bench/data/naturalearth-land.json')
             .then(response => response.json())
@@ -40,12 +45,12 @@ export default class PaintStates extends Benchmark {
                 });
             })
             .then((style) => {
-                return  createMap({
+                return createMap({
                     zoom,
                     width,
                     height,
-                    center: [-77.032194, 38.912753],
-                    style: style
+                    center: this.center,
+                    style
                 }).then(map => {
                     this.map = map;
                 });

diff --git a/bench/benchmarks/query_box.js b/bench/benchmarks/query_box.js
@@ -1,22 +1,36 @@
+// @flow
 
 import Benchmark from '../lib/benchmark';
 import createMap from '../lib/create_map';
+import type Map from '../../src/ui/map';
 
 const width = 1024;
 const height = 768;
-const zooms = [4, 8, 11, 13, 15, 17];
 
 export default class QueryBox extends Benchmark {
-    setup() {
-        return Promise.all(zooms.map(zoom => {
+    style: string;
+    locations: Array<Object>;
+    maps: Array<Map>;
+
+    constructor(style: string, locations: Array<Object>) {
+        super();
+        this.style = style;
+        this.locations = locations;
+    }
+
+    setup(): Promise<void> {
+        return Promise.all(this.locations.map(location => {
             return createMap({
-                zoom,
+                zoom: location.zoom,
                 width,
                 height,
-                center: [-77.032194, 38.912753],
-                style: 'mapbox://styles/mapbox/streets-v9'
+                center: location.center,
+                style: this.style
+            });
+        }))
+            .then(maps => {
+                this.maps = maps;
             });
-        })).then(maps => { this.maps = maps; });
     }
 
     bench() {

diff --git a/bench/benchmarks/query_point.js b/bench/benchmarks/query_point.js
@@ -1,10 +1,11 @@
+// @flow
 
 import Benchmark from '../lib/benchmark';
 import createMap from '../lib/create_map';
+import type Map from '../../src/ui/map';
 
 const width = 1024;
 const height = 768;
-const zooms = [4, 8, 11, 13, 15, 17];
 
 const points = [];
 const d = 4;
@@ -18,16 +19,29 @@ for (let x = 0; x < d; x++) {
 }
 
 export default class QueryPoint extends Benchmark {
-    setup() {
-        return Promise.all(zooms.map(zoom => {
+    style: string;
+    locations: Array<Object>;
+    maps: Array<Map>;
+
+    constructor(style: string, locations: Array<Object>) {
+        super();
+        this.style = style;
+        this.locations = locations;
+    }
+
+    setup(): Promise<void> {
+        return Promise.all(this.locations.map(location => {
             return createMap({
-                zoom,
+                zoom: location.zoom,
                 width,
                 height,
-                center: [-77.032194, 38.912753],
-                style: 'mapbox://styles/mapbox/streets-v9'
+                center: location.center,
+                style: this.style
+            });
+        }))
+            .then(maps => {
+                this.maps = maps;
             });
-        })).then(maps => { this.maps = maps; });
     }
 
     bench() {

diff --git a/bench/benchmarks/style_layer_create.js b/bench/benchmarks/style_layer_create.js
@@ -1,12 +1,21 @@
+// @flow
 
 import Benchmark from '../lib/benchmark';
-import accessToken from '../lib/access_token';
 import createStyleLayer from '../../src/style/create_style_layer';
 import deref from '../../src/style-spec/deref';
+import { normalizeStyleURL } from '../../src/util/mapbox';
 
 export default class StyleLayerCreate extends Benchmark {
-    setup() {
-        return fetch(`https://api.mapbox.com/styles/v1/mapbox/streets-v9?access_token=${accessToken}`)
+    style: string;
+    layers: Array<Object>;
+
+    constructor(style: string) {
+        super();
+        this.style = style;
+    }
+
+    setup(): Promise<void> {
+        return fetch(normalizeStyleURL(this.style))
             .then(response => response.json())
             .then(json => { this.layers = deref(json.layers); });
     }