diff --git a/.circleci/config.yml b/.circleci/config.yml index 4111675b594..bc32c551d0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,6 +205,8 @@ jobs: - attach_workspace: at: . - run: yarn run test-render + - store_test_results: + path: test/integration/render-tests - store_artifacts: path: "test/integration/render-tests/index.html" diff --git a/build/rollup_plugins.js b/build/rollup_plugins.js index 2873872f3ba..15c6aaaa3ac 100644 --- a/build/rollup_plugins.js +++ b/build/rollup_plugins.js @@ -9,11 +9,12 @@ import {terser} from 'rollup-plugin-terser'; import minifyStyleSpec from './rollup_plugin_minify_style_spec'; import {createFilter} from 'rollup-pluginutils'; import strip from '@rollup/plugin-strip'; +import replace from 'rollup-plugin-replace'; // Common set of plugins/transformations shared across different rollup // builds (main mapboxgl bundle, style-spec package, benchmarks bundle) -export const plugins = (minified, production) => [ +export const plugins = (minified, production, test) => [ flow(), minifyStyleSpec(), json(), @@ -21,8 +22,12 @@ export const plugins = (minified, production) => [ sourceMap: true, functions: ['PerformanceUtils.*', 'Debug.*'] }) : false, + test ? replace({ + 'process.env.CI': JSON.stringify(process.env.CI), + 'process.env.UPDATE': JSON.stringify(process.env.UPDATE) + }) : false, glsl('./src/shaders/*.glsl', production), - buble({transforms: {dangerousForOf: true}, objectAssign: "Object.assign"}), + buble({transforms: {dangerousForOf: true, asyncAwait: !test}, objectAssign: "Object.assign"}), minified ? terser({ compress: { pure_getters: true, diff --git a/build/test/build-tape.js b/build/test/build-tape.js index 78f11d8d1e4..cd558c12ce3 100644 --- a/build/test/build-tape.js +++ b/build/test/build-tape.js @@ -6,7 +6,7 @@ const fs = require('fs'); module.exports = function() { return new Promise((resolve, reject) => { browserify(require.resolve('../../test/util/tape_config.js'), { standalone: 'tape' }) - .transform("babelify", {presets: ["@babel/preset-env"], global: true}) + .transform("babelify", {presets: ["@babel/preset-env"], global: true, compact: true}) .bundle((err, buff) => { if (err) { throw err; } diff --git a/package.json b/package.json index 37be37322e0..545d7575a33 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,6 @@ "nyc": "^13.3.0", "pirates": "^4.0.1", "pixelmatch": "^5.1.0", - "pngjs": "^3.4.0", "postcss-cli": "^6.1.2", "postcss-inline-svg": "^3.1.1", "pretty-bytes": "^5.1.0", @@ -110,7 +109,6 @@ "stylelint": "^9.10.1", "stylelint-config-standard": "^18.2.0", "tap": "~12.4.1", - "tap-parser": "^10.0.1", "tape": "^4.13.2", "tape-filter": "^1.0.4", "testem": "^3.0.0" @@ -127,7 +125,7 @@ "build-prod": "rollup -c --environment BUILD:production", "build-prod-min": "rollup -c --environment BUILD:production,MINIFY:true", "build-csp": "rollup -c rollup.config.csp.js", - "build-query-suite": "rollup -c test/integration/rollup.config.test.js", + "build-test-suite": "rollup -c test/integration/rollup.config.test.js", "build-flow-types": "mkdir -p dist && cp build/mapbox-gl.js.flow dist/mapbox-gl.js.flow && cp build/mapbox-gl.js.flow dist/mapbox-gl-dev.js.flow", "build-css": "postcss -o dist/mapbox-gl.css src/css/mapbox-gl.css", "build-style-spec": "cd src/style-spec && npm run build && cd ../.. && mkdir -p dist/style-spec && cp src/style-spec/dist/* dist/style-spec", @@ -136,9 +134,8 @@ "build-benchmarks": "BENCHMARK_VERSION=${BENCHMARK_VERSION:-\"$(git rev-parse --abbrev-ref HEAD) $(git rev-parse --short=7 HEAD)\"} rollup -c bench/versions/rollup_config_benchmarks.js", "watch-benchmarks": "BENCHMARK_VERSION=${BENCHMARK_VERSION:-\"$(git rev-parse --abbrev-ref HEAD) $(git rev-parse --short=7 HEAD)\"} rollup -c bench/rollup_config_benchmarks.js -w", "start-server": "st --no-cache -H 0.0.0.0 --port 9966 --index index.html .", - "start": "run-p build-token watch-css watch-query watch-benchmarks start-server", + "start": "run-p build-token watch-css watch-dev watch-benchmarks start-server", "start-debug": "run-p build-token watch-css watch-dev start-server", - "start-tests": "run-p build-token watch-css watch-query start-server", "start-bench": "run-p build-token watch-benchmarks start-server", "start-release": "run-s build-token build-prod-min build-css print-release-url start-server", "diff-tarball": "build/run-node build/diff-tarball && echo \"Please confirm the above is correct [y/n]? \"; read answer; if [ \"$answer\" = \"${answer#[Yy]}\" ]; then false; fi", @@ -152,10 +149,10 @@ "test-unit": "build/run-tap --reporter classic --no-coverage test/unit", "test-build": "build/run-tap --no-coverage test/build/**/*.test.js", "test-browser": "build/run-tap --reporter spec --no-coverage test/browser/**/*.test.js", - "test-render": "node --max-old-space-size=2048 test/render.test.js", - "test-query-node": "node test/query.test.js", - "watch-query": "testem -f test/integration/testem.js", - "test-query": "testem ci -f test/integration/testem.js -R xunit > test/integration/query-tests/test-results.xml", + "watch-render": "SUITE_NAME=render testem -f test/integration/testem.js", + "watch-query": "SUITE_NAME=query testem -f test/integration/testem.js", + "test-render": "CI=true SUITE_NAME=render testem ci -f test/integration/testem.js", + "test-query": "CI=true SUITE_NAME=query testem ci -f test/integration/testem.js", "test-expressions": "build/run-node test/expression.test.js", "test-flow": "build/run-node build/generate-flow-typed-style-spec && flow .", "test-cov": "nyc --require=@mapbox/flow-remove-types/register --reporter=text-summary --reporter=lcov --cache run-s test-unit test-expressions test-query test-render", diff --git a/src/index.js b/src/index.js index bf55e98f71d..6f29e7ad67f 100644 --- a/src/index.js +++ b/src/index.js @@ -27,6 +27,7 @@ import {prewarm, clearPrewarmedResources} from './util/global_worker_pool'; import {clearTileCache} from './util/tile_request_cache'; import {PerformanceUtils} from './util/performance'; import {FreeCameraOptions} from './ui/free_camera'; +import browser from './util/browser'; const exported = { version, @@ -176,7 +177,7 @@ const exported = { }; //This gets automatically stripped out in production builds. -Debug.extend(exported, {isSafari, getPerformanceMetrics: PerformanceUtils.getPerformanceMetrics}); +Debug.extend(exported, {isSafari, getPerformanceMetrics: PerformanceUtils.getPerformanceMetrics, setNow: browser.setNow, restoreNow: browser.restoreNow}); /** * The version of Mapbox GL JS in use as specified in `package.json`, diff --git a/src/util/browser.js b/src/util/browser.js index 007faee8072..204be58e175 100755 --- a/src/util/browser.js +++ b/src/util/browser.js @@ -31,6 +31,14 @@ const exported = { */ now, + setNow(time: number) { + exported.now = () => time; + }, + + restoreNow() { + exported.now = now; + }, + frame(fn: (paintStartTimestamp: number) => void): Cancelable { const frame = raf(fn); return {cancel: () => cancel(frame)}; diff --git a/test/ajax_stubs.js b/test/ajax_stubs.js deleted file mode 100644 index bda7da8afd0..00000000000 --- a/test/ajax_stubs.js +++ /dev/null @@ -1,133 +0,0 @@ - -import {PNG} from 'pngjs'; -import request from 'request'; -// we're using a require hook to load this file instead of src/util/ajax.js, -// so we import browser module as if it were in an adjacent file -import browser from './browser'; // eslint-disable-line import/no-unresolved -const cache = {}; - -/** - * The type of a resource. - * @private - * @readonly - * @enum {string} - */ -const ResourceType = { - Unknown: 'Unknown', - Style: 'Style', - Source: 'Source', - Tile: 'Tile', - Glyphs: 'Glyphs', - SpriteImage: 'SpriteImage', - SpriteJSON: 'SpriteJSON', - Image: 'Image' -}; -export {ResourceType}; - -if (typeof Object.freeze == 'function') { - Object.freeze(ResourceType); -} - -function cached(data, callback) { - setImmediate(() => { - callback(null, data); - }); -} - -export const getReferrer = () => undefined; - -export const getJSON = function({url}, callback) { - if (cache[url]) return cached(cache[url], callback); - return request(url, (error, response, body) => { - if (!error && response.statusCode >= 200 && response.statusCode < 300) { - let data; - try { - data = JSON.parse(body); - } catch (err) { - return callback(err); - } - cache[url] = data; - callback(null, data); - } else { - callback(error || new Error(response.statusCode)); - } - }); -}; - -export const getArrayBuffer = function({url}, callback) { - if (cache[url]) return cached(cache[url], callback); - return request({url, encoding: null}, (error, response, body) => { - if (!error && response.statusCode >= 200 && response.statusCode < 300) { - cache[url] = body; - callback(null, body); - } else { - if (!error) error = {status: +response.statusCode}; - callback(error); - } - }); -}; - -export const makeRequest = getArrayBuffer; - -export const postData = function({url, body}, callback) { - return request.post(url, body, (error, response, body) => { - if (!error && response.statusCode >= 200 && response.statusCode < 300) { - callback(null, body); - } else { - callback(error || new Error(response.statusCode)); - } - }); -}; - -export const getImage = function({url}, callback) { - if (cache[url]) return cached(cache[url], callback); - const req = request({url, encoding: null}, (error, response, body) => { - if (!error && response.statusCode >= 200 && response.statusCode < 300) { - new PNG().parse(body, (err, png) => { - if (err) return callback(err); - cache[url] = png; - callback(null, png); - }); - } else { - callback(error || {status: response.statusCode}); - } - }); - if (!req.cancel) req.cancel = () => {}; - return req; -}; - -browser.getImageData = function({width, height, data}, padding = 0) { - const source = new Uint8Array(data); - const dest = new Uint8Array((2 * padding + width) * (2 * padding + height) * 4); - - const dstPad = padding > 0 ? padding : 0; - const srcPad = padding >= 0 ? 0 : -padding; - const dstWidth = width + 2 * padding; - for (let i = srcPad; i < height - srcPad; i++) { - const dstRow = i - srcPad + dstPad; - dest.set(source.slice((i * width + srcPad) * 4, ((i + 1) * width - srcPad) * 4), 4 * (dstRow * dstWidth + dstPad)); - } - return {width: dstWidth, height: height + 2 * padding, data: dest}; -}; - -// Hack: since node doesn't have any good video codec modules, just grab a png with -// the first frame and fake the video API. -export const getVideo = function(urls, callback) { - return request({url: urls[0], encoding: null}, (error, response, body) => { - if (!error && response.statusCode >= 200 && response.statusCode < 300) { - new PNG().parse(body, (err, png) => { - if (err) return callback(err); - callback(null, { - readyState: 4, // HAVE_ENOUGH_DATA - addEventListener() {}, - play() {}, - width: png.width, - height: png.height, - data: png.data - }); - }); - } else { - callback(error || new Error(response.statusCode)); - } - }); -}; diff --git a/test/ignores.json b/test/ignores.json index 9a4d41e1a1a..b71e4599d50 100644 --- a/test/ignores.json +++ b/test/ignores.json @@ -1,45 +1,48 @@ { "query-tests/regressions/mapbox-gl-js#4494": "https://github.com/mapbox/mapbox-gl-js/issues/2716", - "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary", - "render-tests/line-dasharray/case/square": "https://github.com/mapbox/mapbox-gl-js/issues/9531", + "render-tests/debug/tile": "skip - inconsistent text rendering with canvas on different platforms", + "render-tests/debug/tile-overscaled": "skip - inconsistent text rendering with canvas on different platforms", + "render-tests/fill-extrusion-pattern/1.5x-on-1x-add-image": "skip - non-deterministic on AMD graphics cards", + "render-tests/fill-extrusion-pattern/multiple-layers-flat": "skip - https://github.com/mapbox/mapbox-gl-js-internal/issues/223", + "render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border": "skip - https://github.com/mapbox/mapbox-gl-js-internal/issues/223", + "render-tests/fill-extrusion-pattern/tile-buffer": "skip - not rendering correctly on CI", + "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-js/issues/7207", + "render-tests/geojson/inline-linestring-fill": "skip - current behavior is arbitrary", + "render-tests/icon-image/icon-sdf-non-sdf-one-layer": "skip - render sdf icon and normal icon in one layer", + "render-tests/icon-text-fit/text-variable-anchor-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/line-dasharray/case/square": "skip - https://github.com/mapbox/mapbox-gl-js/issues/9531", "render-tests/map-mode/static": "https://github.com/mapbox/mapbox-gl-js/issues/5649", "render-tests/map-mode/tile": "skip - mapbox-gl-js does not support tile-mode", "render-tests/map-mode/tile-avoid-edges": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/mixed-zoom/z10-z11": "current behavior conflicts with https://github.com/mapbox/mapbox-gl-js/pull/6803. can be fixed when https://github.com/mapbox/api-maps/issues/1480 is done", "render-tests/projection/axonometric": "axonometric rendering in gl-js tbd", "render-tests/projection/axonometric-multiple": "axonometric rendering in gl-js tbd", "render-tests/projection/skew": "axonometric rendering in gl-js tbd", - "render-tests/regressions/mapbox-gl-js#3682": "skip - true", - "render-tests/runtime-styling/image-update-icon": "skip - https://github.com/mapbox/mapbox-gl-js/issues/4804", - "render-tests/runtime-styling/image-update-pattern": "skip - https://github.com/mapbox/mapbox-gl-js/issues/4804", - "render-tests/mixed-zoom/z10-z11": "current behavior conflicts with https://github.com/mapbox/mapbox-gl-js/pull/6803. can be fixed when https://github.com/mapbox/api-maps/issues/1480 is done", - "render-tests/fill-extrusion-pattern/tile-buffer": "https://github.com/mapbox/mapbox-gl-js/issues/4403", - "render-tests/symbol-placement/line-center-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", "render-tests/symbol-placement/line-center-buffer-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/symbol-placement/line-center-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", "render-tests/symbol-sort-key/text-ignore-placement": "skip - text drawn over icons", - "render-tests/text-variable-anchor/remember-last-placement": "skip - not sure this is correct behavior", - "render-tests/icon-image/icon-sdf-non-sdf-one-layer": "skip - render sdf icon and normal icon in one layer", - "render-tests/text-variable-anchor/all-anchors-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-js/issues/7207", - "render-tests/text-size/zero": "https://github.com/mapbox/mapbox-gl-js/issues/9161", - "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/tile-mode/streets-v11": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/within/paint-line": "https://github.com/mapbox/mapbox-gl-js/issues/7023", - "render-tests/icon-text-fit/text-variable-anchor-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/text-variable-anchor/all-anchors-labels-priority-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", - "render-tests/fill-extrusion-pattern/1.5x-on-1x-add-image": "skip - non-deterministic on AMD graphics cards", - "render-tests/text-variable-anchor/avoid-edges-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", "render-tests/text-font-metrics/font-with-baseline-font-without-baseline": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", + "render-tests/text-font-metrics/font-with-image-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", + "render-tests/text-font-metrics/latin-alphabets-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", + "render-tests/text-font-metrics/latin-alphabets-vertical-no-ascender-descender": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", + "render-tests/text-font-metrics/line-placement-vertical-shaping-with-punctuations": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76", "render-tests/text-font-metrics/mixed-fonts-both-with-baseline": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", "render-tests/text-font-metrics/mixed-fonts-with-image": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", "render-tests/text-font-metrics/mixed-fonts-with-image-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", - "render-tests/text-font-metrics/mixed-fonts-with-scales": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", - "render-tests/text-font-metrics/font-with-image-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", - "render-tests/text-font-metrics/latin-alphabets-vertical-no-ascender-descender": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", - "render-tests/text-font-metrics/latin-alphabets-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", "render-tests/text-font-metrics/mixed-fonts-with-images-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", - "render-tests/text-font-metrics/punctuations-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", - "render-tests/text-font-metrics/line-placement-vertical-shaping-with-punctuations": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76", + "render-tests/text-font-metrics/mixed-fonts-with-scales": "https://github.com/mapbox/mapbox-gl-js-internal/issues/74", "render-tests/text-font-metrics/multiline-point-placement-vertical-shaping-with-punctuations": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76", "render-tests/text-font-metrics/point-placement-vertical-shaping-with-punctuations": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76", - "render-tests/text-font-metrics/vertical-shaping-with-ZWSP": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76" + "render-tests/text-font-metrics/punctuations-vertical": "https://github.com/mapbox/mapbox-gl-js-internal/issues/75", + "render-tests/text-font-metrics/vertical-shaping-with-ZWSP": "https://github.com/mapbox/mapbox-gl-js-internal/issues/76", + "render-tests/text-variable-anchor/all-anchors-labels-priority-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/text-variable-anchor/all-anchors-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/text-variable-anchor/avoid-edges-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/text-variable-anchor/pitched-rotated-debug": "skip - non-deterministic when rendered in browser", + "render-tests/text-variable-anchor/pitched-with-map": "skip - non-deterministic when rendered in browser", + "render-tests/text-variable-anchor/remember-last-placement": "skip - not sure this is correct behavior", + "render-tests/text-size/zero": "https://github.com/mapbox/mapbox-gl-js/issues/9161", + "render-tests/tile-mode/streets-v11": "skip - mapbox-gl-js does not support tile-mode", + "render-tests/within/paint-line": "https://github.com/mapbox/mapbox-gl-js/issues/7023" } diff --git a/test/integration/README.md b/test/integration/README.md index 6046e7c56e9..a64ecf32dda 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -29,7 +29,7 @@ yarn run test-render ``` or ``` -yarn run test-query-node +yarn run test-query ``` To run only the expression tests: @@ -42,7 +42,7 @@ yarn run test-expressions To run a subset of tests or an individual test, you can pass a specific subdirectory to the `test-render` script. For example, to run all the tests for a given property, e.g. `circle-radius`: ``` -$ yarn run test-render circle-radius +$ yarn run test-render tests=circle-radius ... * passed circle-radius/antimeridian * passed circle-radius/default @@ -56,7 +56,7 @@ Done in 2.71s. ``` Or to run a single test: ``` -$ yarn run test-render circle-radius/literal +$ yarn run test-render tests=circle-radius/literal ... * passed circle-radius/literal 1 passed (100.0%) @@ -66,7 +66,7 @@ Done in 2.32s. ### Viewing test results -During a test run, the test harness will use GL-JS to create an `actual.png` image from the given `style.json`, and will then use [pixelmatch](https://github.com/mapbox/pixelmatch) to compare that image to `expected.png`, generating a `diff.png` highlighting the mismatching pixels (if any) in red. +During a test run, the test harness will use GL-JS to create an `actual.png` image from the given `style.json`, and will then use [pixelmatch](https://github.com/mapbox/pixelmatch) to compare that image to `expected.png`, generating a `diff.png` highlighting the mismatched pixels (if any) in red. After the test(s) have run, you can view the results graphically by opening the `index.html` file generated by the harness: @@ -80,18 +80,13 @@ open ./test/integration/query-tests/index.html ## Running tests in the browser -Query tests can be run in the browser, the server for serving up the test page and test fixtures starts when you run +Render and query tests can be run in the browser. The server for serving up the test page and test fixtures starts when you run ``` -yarn run start -``` -OR -``` -yarn run start-debug +yarn run watch-query ``` - -If you want to run only the test server run: +or ``` -yarn run watch-query +yarn run watch-render ``` Then open the following url in the browser of your choice to start running the tests. @@ -107,6 +102,11 @@ A filter can be specified by using the `filter` query param in the url. E.g, add ``` to the end of the url will only run the tests that contain `circle-pitch` in the name. +You can run a specific test by as follows +``` +?filter=circle-radius/antimeridian +``` + ### Build Notifications The terminal window can be very noisy with both the build and the test servers running in the same session. @@ -115,7 +115,6 @@ So the server uses platform notifications to inform when the build has finished. DISABLE_BUILD_NOTIFICATIONS=true ``` - ## Writing new tests _Note: Expected results are always generated with the **js** implementation. This is merely for consistency and does not diff --git a/test/integration/lib/generate-fixture-json.js b/test/integration/lib/generate-fixture-json.js index b4bf77377ff..355b013531b 100644 --- a/test/integration/lib/generate-fixture-json.js +++ b/test/integration/lib/generate-fixture-json.js @@ -4,8 +4,6 @@ const fs = require('fs'); const glob = require('glob'); const localizeURLs = require('./localize-urls'); -const OUTPUT_FILE = 'fixtures.json'; - exports.generateFixtureJson = generateFixtureJson; exports.getAllFixtureGlobs = getAllFixtureGlobs; @@ -47,7 +45,7 @@ function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 't allFiles[fixturePath] = json; } else if (extension === '.png') { - allFiles[fixturePath] = pngToBase64Str(fixturePath); + allFiles[fixturePath] = true; } else { throw new Error(`${extension} is incompatible , file path ${fixturePath}`); } @@ -64,7 +62,7 @@ function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 't //Skip if test is malformed if (malformedTests[testName]) { continue; } - //Lazily initaialize an object to store each file wihin a particular testName + //Lazily initialize an object to store each file wihin a particular testName if (result[testName] == null) { result[testName] = {}; } @@ -74,7 +72,8 @@ function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 't } const outputStr = JSON.stringify(result, null, 4); - const outputPath = path.join(outputDirectory, OUTPUT_FILE); + const outputFile = `${suiteDirectory.split('-')[0]}-fixtures.json`; + const outputPath = path.join(outputDirectory, outputFile); return new Promise((resolve, reject) => { fs.writeFile(outputPath, outputStr, {encoding: 'utf8'}, (err) => { @@ -87,7 +86,7 @@ function generateFixtureJson(rootDirectory, suiteDirectory, outputDirectory = 't function getAllFixtureGlobs(rootDirectory, suiteDirectory) { const basePath = path.join(rootDirectory, suiteDirectory); - const jsonPaths = path.join(basePath, '/**/*.json'); + const jsonPaths = path.join(basePath, '/**/[!actual]*.json'); const imagePaths = path.join(basePath, '/**/*.png'); return [jsonPaths, imagePaths]; @@ -97,10 +96,6 @@ function parseJsonFromFile(filePath) { return JSON.parse(fs.readFileSync(filePath, {encoding: 'utf8'})); } -function pngToBase64Str(filePath) { - return fs.readFileSync(filePath).toString('base64'); -} - function processStyle(testName, style) { const clone = JSON.parse(JSON.stringify(style)); // 7357 is testem's default port diff --git a/test/integration/lib/operation-handlers.js b/test/integration/lib/operation-handlers.js index d653c892943..8d74069715a 100644 --- a/test/integration/lib/operation-handlers.js +++ b/test/integration/lib/operation-handlers.js @@ -1,4 +1,9 @@ -function handleOperation(map, operations, opIndex, doneCb) { +/* eslint-env browser */ +/* global mapboxgl:readonly */ +import customLayerImplementations from '../custom_layer_implementations'; + +function handleOperation(map, options, opIndex, doneCb) { + const operations = options.operations; const operation = operations[opIndex]; const opName = operation[0]; //Delegate to special handler if one is available @@ -8,6 +13,11 @@ function handleOperation(map, operations, opIndex, doneCb) { }); } else { map[opName](...operation.slice(1)); + // Render one more frame with forceDrapeFirst + if (options.terrainDrapeFirst && map.painter.terrain) { + map.painter.terrain.forceDrapeFirst = true; + map._render(); + } doneCb(opIndex); } } @@ -15,10 +25,17 @@ function handleOperation(map, operations, opIndex, doneCb) { export const operationHandlers = { wait(map, params, doneCb) { const wait = function() { - if (map.loaded()) { + if (params.length) { + window._renderTestNow += params[0]; + mapboxgl.setNow(window._renderTestNow); + map._render(); doneCb(); } else { - map.once('render', wait); + if (map.loaded()) { + doneCb(); + } else { + map.once('render', wait); + } } }; wait(); @@ -32,12 +49,75 @@ export const operationHandlers = { } }; idle(); + }, + sleep(map, params, doneCb) { + setTimeout(doneCb, params[0]); + }, + addImage(map, params, doneCb) { + const image = new Image(); + image.onload = () => { + map.addImage(params[0], image, params[2] || {}); + doneCb(); + }; + + image.src = params[1].replace('./', ''); + image.onerror = () => { + throw new Error(`addImage opertation failed with src ${image.src}`); + }; + }, + addCustomLayer(map, params, doneCb) { + map.addLayer(new customLayerImplementations[params[0]](), params[1]); + map._render(); + doneCb(); + }, + updateFakeCanvas(map, params, doneCb) { + const updateFakeCanvas = async function() { + const canvasSource = map.getSource(params[0]); + canvasSource.play(); + // update before pause should be rendered + await updateCanvas(params[1]); + canvasSource.pause(); + await updateCanvas(params[2]); + map._render(); + doneCb(); + }; + updateFakeCanvas(); + }, + setStyle(map, params, doneCb) { + // Disable local ideograph generation (enabled by default) for + // consistent local ideograph rendering using fixtures in all runs of the test suite. + map.setStyle(params[0], {localIdeographFontFamily: false}); + doneCb(); + }, + pauseSource(map, params, doneCb) { + map.style.sourceCaches[params[0]].pause(); + doneCb(); + }, + setCameraPosition(map, params, doneCb) { + const options = map.getFreeCameraOptions(); + const location = params[0]; // lng, lat, altitude + options.position = mapboxgl.MercatorCoordinate.fromLngLat(new mapboxgl.LngLat(location[0], location[1]), location[2]); + map.setFreeCameraOptions(options); + doneCb(); + }, + lookAtPoint(map, params, doneCb) { + const options = map.getFreeCameraOptions(); + const location = params[0]; + const upVector = params[1]; + options.lookAtPoint(new mapboxgl.LngLat(location[0], location[1]), upVector); + map.setFreeCameraOptions(options); + doneCb(); } }; -export function applyOperations(map, operations, doneCb) { - // No operations specified, end immediately adn invoke doneCb. +export function applyOperations(map, options, doneCb) { + const operations = options.operations; + // No operations specified, end immediately and invoke doneCb. if (!operations || operations.length === 0) { + if (options.terrainDrapeFirst && map.painter.terrain) { + map.painter.terrain.forceDrapeFirst = true; + map._render(); // Render one more time with forceDrapeFirst. + } doneCb(); return; } @@ -50,7 +130,23 @@ export function applyOperations(map, operations, doneCb) { return; } - handleOperation(map, operations, ++lastOpIndex, scheduleNextOperation); + handleOperation(map, options, ++lastOpIndex, scheduleNextOperation); }; scheduleNextOperation(-1); } + +function updateCanvas(imagePath) { + return new Promise((resolve) => { + const canvas = window.document.getElementById('fake-canvas'); + const ctx = canvas.getContext('2d'); + const image = new Image(); + image.src = imagePath.replace('./', ''); + image.onload = () => { + resolve(ctx.drawImage(image, 0, 0, image.width, image.height)); + }; + + image.onerror = () => { + throw new Error(`updateFakeCanvas failed to load image at ${image.src}`); + }; + }); +} diff --git a/test/integration/lib/query-browser.js b/test/integration/lib/query-browser.js deleted file mode 100644 index 4eb275a70af..00000000000 --- a/test/integration/lib/query-browser.js +++ /dev/null @@ -1,86 +0,0 @@ -/* eslint-env browser */ -/* global tape:readonly, mapboxgl:readonly */ -/* eslint-disable import/no-unresolved */ -// fixtures.json is automatically generated before this file gets built -// refer testem.js#before_tests() -import fixtures from '../dist/fixtures.json'; -import ignores from '../../ignores.json'; -import {applyOperations} from './operation-handlers'; -import {deepEqual, generateDiffLog} from './json-diff'; - -for (const testName in fixtures) { - if (testName in ignores) { - tape.skip(testName, testFunc); - } else { - tape(testName, {timeout: 20000}, testFunc); - } -} - -function testFunc(t) { - // This needs to be read from the `t` object because this function runs async in a closure. - const currentTestName = t.name; - const style = fixtures[currentTestName].style; - const expected = fixtures[currentTestName].expected; - const options = style.metadata.test; - const skipLayerDelete = style.metadata.skipLayerDelete; - - window.devicePixelRatio = options.pixelRatio; - - //1. Create and position the container, floating at the bottom right - const container = document.createElement('div'); - container.style.position = 'fixed'; - container.style.bottom = '10px'; - container.style.right = '10px'; - container.style.width = `${options.width}px`; - container.style.height = `${options.height}px`; - document.body.appendChild(container); - - //2. Initialize the Map - const map = new mapboxgl.Map({ - container, - style, - classes: options.classes, - interactive: false, - attributionControl: false, - preserveDrawingBuffer: true, - axonometric: options.axonometric || false, - skew: options.skew || [0, 0], - fadeDuration: options.fadeDuration || 0, - localIdeographFontFamily: options.localIdeographFontFamily || false, - crossSourceCollisions: typeof options.crossSourceCollisions === "undefined" ? true : options.crossSourceCollisions - }); - map.repaint = true; - map.once('load', () => { - //3. Run the operations on the map - applyOperations(map, options.operations, () => { - - //4. Perform query operation and compare results from expected values - const results = options.queryGeometry ? - map.queryRenderedFeatures(options.queryGeometry, options.queryOptions || {}) : - []; - - const actual = results.map((feature) => { - const featureJson = JSON.parse(JSON.stringify(feature.toJSON())); - if (!skipLayerDelete) delete featureJson.layer; - return featureJson; - }); - - const testMetaData = { - name: t.name, - actual: map.getCanvas().toDataURL() - }; - const success = deepEqual(actual, expected); - if (success) { - t.pass(JSON.stringify(testMetaData)); - } else { - testMetaData['difference'] = generateDiffLog(expected, actual); - t.fail(JSON.stringify(testMetaData)); - } - //Cleanup WebGL context - map.remove(); - delete map.painter.context.gl; - document.body.removeChild(container); - t.end(); - }); - }); -} diff --git a/test/integration/lib/query.js b/test/integration/lib/query.js index 5485d39d078..8f71bfa6946 100644 --- a/test/integration/lib/query.js +++ b/test/integration/lib/query.js @@ -1,153 +1,127 @@ -import path from 'path'; -import fs from 'fs'; -import * as diff from 'diff'; -import {PNG} from 'pngjs'; -import harness from './harness'; - -function deepEqual(a, b) { - if (typeof a !== typeof b) - return false; - if (typeof a === 'number') - return Math.abs(a - b) < 1e-10; - if (a === null || typeof a !== 'object') - return a === b; - - const ka = Object.keys(a); - const kb = Object.keys(b); - - if (ka.length !== kb.length) - return false; - - ka.sort(); - kb.sort(); - - for (let i = 0; i < ka.length; i++) - if (ka[i] !== kb[i] || !deepEqual(a[ka[i]], b[ka[i]])) - return false; - - return true; +/* eslint-env browser */ +/* global tape:readonly, mapboxgl:readonly */ +/* eslint-disable import/no-unresolved */ +// query-fixtures.json is automatically generated before this file gets built +// refer testem.js#before_tests() +import fixtures from '../dist/query-fixtures.json'; +import ignores from '../../ignores.json'; +import {applyOperations} from './operation-handlers'; +import {deepEqual, generateDiffLog} from './json-diff'; +import {setupHTML, updateHTML} from '../../util/html_generator.js'; + +window._suiteName = 'query-tests'; +setupHTML(); + +const browserWriteFile = new Worker('../util/browser_write_file.js'); + +for (const testName in fixtures) { + const options = {timeout: 20000}; + if (testName in ignores) { + const ignoreType = ignores[testName]; + if (/^skip/.test(ignoreType)) { + options.skip = true; + } else { + options.todo = true; + } + } + + tape(testName, options, testFunc); } -/** - * Run the query suite. - * - * @param {string} implementation - identify the implementation under test; used to - * deal with implementation-specific test exclusions and fudge-factors - * @param {Object} options - * @param {Array} [options.tests] - array of test names to run; tests not in the - * array will be skipped - * @param {queryFn} query - a function that performs the query - * @returns {undefined} terminates the process when testing is complete - */ -export function run(implementation, options, query) { - const directory = path.join(__dirname, '../query-tests'); - harness(directory, implementation, options, (style, params, done) => { - query(style, params, (err, data, results) => { - if (err) return done(err); - - const dir = path.join(directory, params.id); +function testFunc(t) { + // This needs to be read from the `t` object because this function runs async in a closure. + const currentTestName = t.name; + const writeFileBasePath = `test/integration/${currentTestName}`; + const style = fixtures[currentTestName].style; + const expected = fixtures[currentTestName].expected || ''; + const options = style.metadata.test; + const skipLayerDelete = style.metadata.skipLayerDelete; + + window.devicePixelRatio = options.pixelRatio; + + //1. Create and position the container, floating at the bottom right + const container = document.createElement('div'); + container.style.position = 'fixed'; + container.style.bottom = '10px'; + container.style.right = '10px'; + container.style.width = `${options.width}px`; + container.style.height = `${options.height}px`; + document.body.appendChild(container); + + //2. Initialize the Map + const map = new mapboxgl.Map({ + container, + style, + classes: options.classes, + interactive: false, + attributionControl: false, + preserveDrawingBuffer: true, + axonometric: options.axonometric || false, + skew: options.skew || [0, 0], + fadeDuration: options.fadeDuration || 0, + localIdeographFontFamily: options.localIdeographFontFamily || false, + crossSourceCollisions: typeof options.crossSourceCollisions === "undefined" ? true : options.crossSourceCollisions + }); + map.repaint = true; + map.once('load', () => { + //3. Run the operations on the map + applyOperations(map, options, () => { + + //4. Perform query operation and compare results from expected values + const results = options.queryGeometry ? + map.queryRenderedFeatures(options.queryGeometry, options.queryOptions || {}) : + []; + + const actual = results.map((feature) => { + const featureJson = JSON.parse(JSON.stringify(feature.toJSON())); + if (!skipLayerDelete) delete featureJson.layer; + return featureJson; + }); - if (process.env.UPDATE) { - fs.writeFile(path.join(dir, 'expected.json'), JSON.stringify(results, null, 2), done); - return; - } + const testMetaData = { + name: t.name, + actual: map.getCanvas().toDataURL() + }; + const success = deepEqual(actual, expected); + const jsonDiff = generateDiffLog(expected, actual); - const expected = require(path.join(dir, 'expected.json')); - - params.ok = deepEqual(results, expected); - - if (!params.ok) { - const msg = diff.diffJson(expected, results) - .map((hunk) => { - if (hunk.added) { - return `+ ${hunk.value}`; - } else if (hunk.removed) { - return `- ${hunk.value}`; - } else { - return ` ${hunk.value}`; - } - }) - .join(''); - - params.difference = msg; - console.log(msg); + if (!success) { + testMetaData['jsonDiff'] = jsonDiff; } + t.ok(success || t._todo, t.name); + testMetaData.status = t._todo ? 'todo' : success ? 'passed' : 'failed'; - const width = params.width * params.pixelRatio; - const height = params.height * params.pixelRatio; - - const color = [255, 0, 0, 255]; + updateHTML(testMetaData); - function scaleByPixelRatio(x) { - return x * params.pixelRatio; - } + let fileInfo; - if (!Array.isArray(params.queryGeometry[0])) { - const p = params.queryGeometry.map(scaleByPixelRatio); - const d = 30; - drawAxisAlignedLine([p[0] - d, p[1]], [p[0] + d, p[1]], data, width, height, color); - drawAxisAlignedLine([p[0], p[1] - d], [p[0], p[1] + d], data, width, height, color); + if (process.env.UPDATE) { + fileInfo = [ + { + path: `${writeFileBasePath}/expected.json`, + data: jsonDiff.replace('+', '').trim() + } + ]; } else { - const a = params.queryGeometry[0].map(scaleByPixelRatio); - const b = params.queryGeometry[1].map(scaleByPixelRatio); - drawAxisAlignedLine([a[0], a[1]], [a[0], b[1]], data, width, height, color); - drawAxisAlignedLine([a[0], b[1]], [b[0], b[1]], data, width, height, color); - drawAxisAlignedLine([b[0], b[1]], [b[0], a[1]], data, width, height, color); - drawAxisAlignedLine([b[0], a[1]], [a[0], a[1]], data, width, height, color); + fileInfo = [ + { + path: `${writeFileBasePath}/actual.png`, + data: testMetaData.actual.split(',')[1] + }, + { + path: `${writeFileBasePath}/actual.json`, + data: jsonDiff.trim() + } + ]; } - const actualJSON = path.join(dir, 'actual.json'); - fs.writeFile(actualJSON, JSON.stringify(results, null, 2), () => {}); - - const actualPNG = path.join(dir, 'actual.png'); + browserWriteFile.postMessage(fileInfo); - const png = new PNG({ - width: params.width * params.pixelRatio, - height: params.height * params.pixelRatio - }); - - png.data = data; - - png.pack() - .pipe(fs.createWriteStream(actualPNG)) - .on('finish', () => { - params.actual = fs.readFileSync(actualPNG).toString('base64'); - done(); - }); + //Cleanup WebGL context + map.remove(); + delete map.painter.context.gl; + document.body.removeChild(container); + t.end(); }); }); } - -function drawAxisAlignedLine(a, b, pixels, width, height, color) { - const fromX = clamp(Math.min(a[0], b[0]), 0, width); - const toX = clamp(Math.max(a[0], b[0]), 0, width); - const fromY = clamp(Math.min(a[1], b[1]), 0, height); - const toY = clamp(Math.max(a[1], b[1]), 0, height); - - let index; - if (fromX === toX) { - for (let y = fromY; y <= toY; y++) { - index = getIndex(fromX, y); - pixels[index + 0] = color[0]; - pixels[index + 1] = color[1]; - pixels[index + 2] = color[2]; - pixels[index + 3] = color[3]; - } - } else { - for (let x = fromX; x <= toX; x++) { - index = getIndex(x, fromY); - pixels[index + 0] = color[0]; - pixels[index + 1] = color[1]; - pixels[index + 2] = color[2]; - pixels[index + 3] = color[3]; - } - } - - function getIndex(x, y) { - return (y * width + x) * 4; - } -} - -function clamp(x, a, b) { - return Math.max(a, Math.min(b, x)); -} diff --git a/test/integration/lib/render.js b/test/integration/lib/render.js index e2846a47c0b..31e4ca03037 100644 --- a/test/integration/lib/render.js +++ b/test/integration/lib/render.js @@ -1,183 +1,380 @@ -import path from 'path'; -import fs from 'fs'; -import {PNG} from 'pngjs'; -import harness from './harness'; +/* eslint-env browser */ +/* global tape:readonly, mapboxgl:readonly */ +/* eslint-disable import/no-unresolved */ +// render-fixtures.json is automatically generated before this file gets built +// refer testem.js#before_tests() +import fixtures from '../dist/render-fixtures.json'; +import ignores from '../../ignores.json'; +import config from '../../../src/util/config'; +import {clamp} from '../../../src/util/util'; +import {mercatorZfromAltitude} from '../../../src/geo/mercator_coordinate'; +import {setupHTML, updateHTML} from '../../util/html_generator.js'; +import {applyOperations} from './operation-handlers'; import pixelmatch from 'pixelmatch'; -import * as glob from 'glob'; - -/** - * Run the render test suite, compute differences to expected values (making exceptions based on - * implementation vagaries), print results to standard output, write test artifacts to the - * filesystem (optionally updating expected results), and exit the process with a success or - * failure code. - * - * Caller must supply a `render` function that does the actual rendering and passes the raw image - * result on to the `render` function's callback. - * - * A local server is launched that is capable of serving requests for the source, sprite, - * font, and tile assets needed by the tests, and the URLs within the test styles are - * rewritten to point to that server. - * - * As the tests run, results are printed to standard output, and test artifacts are written - * to the filesystem. If the environment variable `UPDATE` is set, the expected artifacts are - * updated in place based on the test rendering. - * - * If all the tests are successful, this function exits the process with exit code 0. Otherwise - * it exits with 1. If an unexpected error occurs, it exits with -1. - * - * @param {string} implementation - identify the implementation under test; used to - * deal with implementation-specific test exclusions and fudge-factors - * @param {Object} [ignores] - map of test names to disable. A key is the relative - * path to a test directory, e.g. `"render-tests/background-color/default"`. A value is a string - * that by convention links to an issue that explains why the test is currently disabled. By default, - * disabled tests will be run, but not fail the test run if the result does not match the expected - * result. If the value begins with "skip", the test will not be run at all -- use this for tests - * that would crash the test harness entirely if they were run. - * @param {renderFn} render - a function that performs the rendering - * @returns {undefined} terminates the process when testing is complete - */ -export function run(implementation, ignores, render) { - const options = {ignores, tests:[], shuffle:false, recycleMap:false, seed:makeHash()}; - - // https://stackoverflow.com/a/1349426/229714 - function makeHash() { - const array = []; - const possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - - for (let i = 0; i < 10; ++i) - array.push(possible.charAt(Math.floor(Math.random() * possible.length))); - - // join array elements without commas. - return array.join(''); - } +import {vec3, vec4} from 'gl-matrix'; + +const browserWriteFile = new Worker('../util/browser_write_file.js'); + +// We are self-hosting test files. +config.REQUIRE_ACCESS_TOKEN = false; +window._suiteName = 'render-tests'; + +mapboxgl.prewarm(); +mapboxgl.setRTLTextPlugin('https://api.mapbox.com/mapbox-gl-js/plugins/mapbox-gl-rtl-text/v0.2.3/mapbox-gl-rtl-text.js'); + +//1. Create and position the container, floating at the bottom right +const container = document.createElement('div'); +container.style.position = 'fixed'; +container.style.bottom = '10px'; +container.style.right = '10px'; +document.body.appendChild(container); + +setupHTML(); + +const {canvas: expectedCanvas, ctx: expectedCtx} = createCanvas(); +const {canvas: diffCanvas, ctx: diffCtx} = createCanvas(); - function checkParameter(param) { - const index = options.tests.indexOf(param); - if (index === -1) - return false; - options.tests.splice(index, 1); - return true; +tape.onFinish(() => { + document.body.removeChild(container); + mapboxgl.clearPrewarmedResources(); +}); + +for (const testName in fixtures) { + const options = {timeout: 20000}; + if (testName in ignores) { + const ignoreType = ignores[testName]; + if (/^skip/.test(ignoreType)) { + options.skip = true; + } else { + options.todo = true; + } } - function checkValueParameter(defaultValue, param) { - const index = options.tests.findIndex((elem) => { return String(elem).startsWith(param); }); - if (index === -1) - return defaultValue; + tape(testName, options, testFunc); +} - const split = String(options.tests.splice(index, 1)).split('='); - if (split.length !== 2) - return defaultValue; +async function testFunc(t) { + // This needs to be read from the `t` object because this function runs async in a closure. + const currentTestName = t.name; + const writeFileBasePath = `test/integration/${currentTestName}`; + const currentFixture = fixtures[currentTestName]; + const style = currentFixture.style; + const options = style.metadata.test; - return split[1]; + // there may be multiple expected images, covering different platforms + const expectedPaths = []; + for (const prop in currentFixture) { + if (prop.indexOf('expected') > -1) { + let path = `${currentTestName}/${prop}.png`; + // regression tests with # in the name need to be sanitized + path = encodeURIComponent(path); + expectedPaths.push(path); + } } - if (process.argv.length > 2) { - options.tests = process.argv.slice(2).filter((value, index, self) => { return self.indexOf(value) === index; }) || []; - options.shuffle = checkParameter('--shuffle'); - options.recycleMap = checkParameter('--recycle-map'); - options.seed = checkValueParameter(options.seed, '--seed'); + const expectedImagePromises = Promise.all(expectedPaths.map((path) => drawImage(expectedCanvas, expectedCtx, path))); + + window.devicePixelRatio = options.pixelRatio; + + if (options.addFakeCanvas) { + const {canvas, ctx} = createCanvas(options.addFakeCanvas.id); + const src = options.addFakeCanvas.image.replace('./', ''); + await drawImage(canvas, ctx, src, false); + window.document.body.appendChild(canvas); } - const directory = path.join(__dirname, '../render-tests'); - harness(directory, implementation, options, (style, params, done) => { - render(style, params, (err, data) => { - if (err) return done(err); - - let stats; - const dir = path.join(directory, params.id); - try { - stats = fs.statSync(dir, fs.R_OK | fs.W_OK); - if (!stats.isDirectory()) throw new Error(); - } catch (e) { - fs.mkdirSync(dir); + container.style.width = `${options.width}px`; + container.style.height = `${options.height}px`; + + //2. Initialize the Map + const map = new mapboxgl.Map({ + container, + style, + classes: options.classes, + interactive: false, + attributionControl: false, + preserveDrawingBuffer: true, + axonometric: options.axonometric || false, + skew: options.skew || [0, 0], + fadeDuration: options.fadeDuration || 0, + localIdeographFontFamily: options.localIdeographFontFamily || false, + crossSourceCollisions: typeof options.crossSourceCollisions === "undefined" ? true : options.crossSourceCollisions, + transformRequest: (url, resourceType) => { + // some tests have the port hardcoded to 2900 + // this makes that backwards compatible + if (resourceType === 'Tile') { + const transformedUrl = new URL(url); + transformedUrl.port = '7357'; + return { + url: transformedUrl.toString() + }; + } + } + }); + map.repaint = true; + + // override internal timing to enable precise wait operations + window._renderTestNow = 0; + mapboxgl.setNow(window._renderTestNow); + + if (options.debug) map.showTileBoundaries = true; + if (options.showOverdrawInspector) map.showOverdrawInspector = true; + if (options.showPadding) map.showPadding = true; + if (options.collisionDebug) map.showCollisionBoxes = true; + + // Disable anisotropic filtering on render tests + map.painter.context.extTextureFilterAnisotropicForceOff = true; + + const gl = map.painter.context.gl; + map.once('load', async () => { + //3. Run the operations on the map + applyOperations(map, options, async () => { + map.repaint = false; + const viewport = gl.getParameter(gl.VIEWPORT); + const w = viewport[2]; + const h = viewport[3]; + let actualImageData; + + // 1. get pixel data from test canvas as Uint8Array + if (options.output === "terrainDepth") { + const pixels = drawTerrainDepth(map, w, h); + actualImageData = Uint8Array.from(pixels); } - const expectedPath = path.join(dir, 'expected.png'); - const actualPath = path.join(dir, 'actual.png'); - const diffPath = path.join(dir, 'diff.png'); - - const width = Math.floor(params.width * params.pixelRatio); - const height = Math.floor(params.height * params.pixelRatio); - const actualImg = new PNG({width, height}); - - // PNG data must be unassociated (not premultiplied) - for (let i = 0; i < data.length; i++) { - const a = data[i * 4 + 3] / 255; - if (a !== 0) { - data[i * 4 + 0] /= a; - data[i * 4 + 1] /= a; - data[i * 4 + 2] /= a; + if (!actualImageData) { + actualImageData = new Uint8Array(gl.drawingBufferWidth * gl.drawingBufferHeight * 4); + gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, actualImageData); + + // readPixels premultiplies the alpha channel so we need to + // undo that for comparison with the expected image pixels + for (let i = 0; i < actualImageData.length; i += 4) { + const alpha = actualImageData[i + 3] / 255; + actualImageData[i + 0] /= alpha; + actualImageData[i + 1] /= alpha; + actualImageData[i + 2] /= alpha; + } + + // readPixels starts at the bottom of the canvas + // so we need to flip the image data + const stride = w * 4; + const temp = new Uint8Array(w * 4); + for (let i = 0; i < (h / 2 | 0); ++i) { + const topOffset = i * stride; + const bottomOffset = (h - i - 1) * stride; + temp.set(actualImageData.subarray(topOffset, topOffset + stride)); + actualImageData.copyWithin(topOffset, bottomOffset, bottomOffset + stride); + actualImageData.set(temp, bottomOffset); } } - actualImg.data = data; - // there may be multiple expected images, covering different platforms - const expectedPaths = glob.sync(path.join(dir, 'expected*.png')); + // if we have multiple expected images, we'll compare against each one and pick the one with + // the least amount of difference; this is useful for covering features that render differently + // depending on platform, i.e. heatmaps use half-float textures for improved rendering where supported + const expectedImages = await expectedImagePromises; - if (!process.env.UPDATE && expectedPaths.length === 0) { - throw new Error('No expected*.png files found; did you mean to run tests with UPDATE=true?'); + if (!process.env.UPDATE && expectedImages.length === 0) { + console.warn('No expected*.png files found; did you mean to run tests with UPDATE=true?'); + t.end(); } - if (process.env.UPDATE) { - fs.writeFileSync(expectedPath, PNG.sync.write(actualImg)); + let fileInfo; + const actual = map.getCanvas().toDataURL(); + if (process.env.UPDATE) { + fileInfo = [ + { + path: `${writeFileBasePath}/expected.png`, + data: actual.split(',')[1] + } + ]; } else { - // if we have multiple expected images, we'll compare against each one and pick the one with - // the least amount of difference; this is useful for covering features that render differently - // depending on platform, i.e. heatmaps use half-float textures for improved rendering where supported + // 2. draw expected.png into a canvas and extract ImageData + let minDiffImage; + let minExpectedCanvas; let minDiff = Infinity; - let minDiffImg, minExpectedBuf; - for (const path of expectedPaths) { - const expectedBuf = fs.readFileSync(path); - const expectedImg = PNG.sync.read(expectedBuf); - const diffImg = new PNG({width, height}); + for (let i = 0; i < expectedImages.length; i++) { + // 3. set up Uint8ClampedArray to write diff into + const diffImage = new Uint8ClampedArray(w * h * 4); - const diff = pixelmatch( - actualImg.data, expectedImg.data, diffImg.data, - width, height, {threshold: 0.1285}) / (width * height); + // 4. Use pixelmatch to compare actual and expected images and write diff + // all inputs must be Uint8Array or Uint8ClampedArray + const currentDiff = pixelmatch(actualImageData, expectedImages[i].data, diffImage, w, h, {threshold: 0.1285}) / (w * h); - if (diff < minDiff) { - minDiff = diff; - minDiffImg = diffImg; - minExpectedBuf = expectedBuf; + if (currentDiff < minDiff) { + minDiff = currentDiff; + minDiffImage = diffImage; + minExpectedCanvas = expectedCanvas; } } - const diffBuf = PNG.sync.write(minDiffImg, {filterType: 4}); - const actualBuf = PNG.sync.write(actualImg, {filterType: 4}); + // 5. Convert diff Uint8Array to ImageData and write to canvas + // so we can get a base64 string to display the diff in the browser + diffCanvas.width = w; + diffCanvas.height = h; + const diffImageData = new ImageData(minDiffImage, w, h); + diffCtx.putImageData(diffImageData, 0, 0); - fs.writeFileSync(diffPath, diffBuf); - fs.writeFileSync(actualPath, actualBuf); + const expected = minExpectedCanvas.toDataURL(); + const imgDiff = diffCanvas.toDataURL(); - params.difference = minDiff; - params.ok = minDiff <= params.allowed; + // 6. use browserWriteFile to write actual and diff to disk (convert image back to base64) + fileInfo = [ + { + path: `${writeFileBasePath}/actual.png`, + data: actual.split(',')[1] + }, + { + path: `${writeFileBasePath}/diff.png`, + data: imgDiff.split(',')[1] + } + ]; + + // 7. pass image paths to testMetaData so the UI can load them from disk + const testMetaData = { + name: currentTestName, + actual, + expected, + imgDiff + }; - params.actual = actualBuf.toString('base64'); - params.expected = minExpectedBuf.toString('base64'); - params.diff = diffBuf.toString('base64'); + const pass = minDiff <= options.allowed; + t.ok(pass || t._todo, t.name); + testMetaData.status = t._todo ? 'todo' : pass ? 'passed' : 'failed'; + updateHTML(testMetaData); } - done(); + browserWriteFile.postMessage(fileInfo); + + //Cleanup WebGL context + map.remove(); + delete map.painter.context.gl; + expectedCtx.clearRect(0, 0, expectedCanvas.width, expectedCanvas.height); + diffCtx.clearRect(0, 0, diffCanvas.width, diffCanvas.height); + + if (options.addFakeCanvas) { + const canvas = window.document.getElementById(options.addFakeCanvas.id); + canvas.parentNode.removeChild(canvas); + } + + mapboxgl.restoreNow(); + t.end(); }); }); } -/** - * @callback renderFn - * @param {Object} style - style to render - * @param {Object} options - * @param {number} options.width - render this wide - * @param {number} options.height - render this high - * @param {number} options.pixelRatio - render with this pixel ratio - * @param {boolean} options.shuffle - shuffle tests sequence - * @param {String} options.seed - Shuffle seed - * @param {boolean} options.recycleMap - trigger map object recycling - * @param {renderCallback} callback - callback to call with the results of rendering - */ - -/** - * @callback renderCallback - * @param {?Error} error - * @param {Buffer} [result] - raw RGBA image data - */ +function drawImage(canvas, ctx, src, getImageData = true) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => { + canvas.height = image.height; + canvas.width = image.width; + if (!getImageData) { + resolve(ctx.drawImage(image, 0, 0)); + } + ctx.drawImage(image, 0, 0); + const result = ctx.getImageData(0, 0, image.width, image.height); + result.src = src; + resolve(result); + }; + image.onerror = reject; + image.src = src; + }); +} + +function createCanvas(id = 'fake-canvas') { + const canvas = window.document.createElement('canvas'); + canvas.id = id; + const ctx = canvas.getContext('2d'); + return {canvas, ctx}; +} + +function drawTerrainDepth(map, width, height) { + if (!map.painter.terrain) + return undefined; + + const terrain = map.painter.terrain; + const tr = map.transform; + const ws = tr.worldSize; + + // Compute frustum corner points in web mercator [0, 1] space where altitude is in meters + const clipSpaceCorners = [ + [-1, 1, -1, 1], + [ 1, 1, -1, 1], + [ 1, -1, -1, 1], + [-1, -1, -1, 1], + [-1, 1, 1, 1], + [ 1, 1, 1, 1], + [ 1, -1, 1, 1], + [-1, -1, 1, 1] + ]; + + const frustumCoords = clipSpaceCorners + .map(v => { + const s = vec4.transformMat4([], v, tr.invProjMatrix); + const k = 1.0 / s[3] / ws; + // Z scale in meters. + return vec4.mul(s, s, [k, k, 1.0 / s[3], k]); + }); + + const nearTL = frustumCoords[0]; + const nearTR = frustumCoords[1]; + const nearBL = frustumCoords[3]; + const farTL = frustumCoords[4]; + const farTR = frustumCoords[5]; + const farBL = frustumCoords[7]; + + // Compute basis vectors X & Y of near and far planes in transformed space. + // These vectors are then interpolated to find corresponding world rays for each screen pixel. + const nearRight = vec3.sub([], nearTR, nearTL); + const nearDown = vec3.sub([], nearBL, nearTL); + const farRight = vec3.sub([], farTR, farTL); + const farDown = vec3.sub([], farBL, farTL); + + const distances = []; + const data = []; + const metersToPixels = mercatorZfromAltitude(1.0, tr.center.lat); + let minDistance = Number.MAX_VALUE; + let maxDistance = 0; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + // Use uv-coordinates of the screen pixel to find positions on near and far planes + const u = (x + 0.5) / width; + const v = (y + 0.5) / height; + + const startPoint = vec3.add([], nearTL, vec3.add([], vec3.scale([], nearRight, u), vec3.scale([], nearDown, v))); + const endPoint = vec3.add([], farTL, vec3.add([], vec3.scale([], farRight, u), vec3.scale([], farDown, v))); + const dir = vec3.normalize([], vec3.sub([], endPoint, startPoint)); + const t = terrain.raycast(startPoint, dir, terrain.exaggeration()); + + if (t !== null) { + // The ray hit the terrain. Compute distance in world space and store to an intermediate array + const point = vec3.scaleAndAdd([], startPoint, dir, t); + const startToPoint = vec3.sub([], point, startPoint); + startToPoint[2] *= metersToPixels; + + const distance = vec3.length(startToPoint) * ws; + distances.push(distance); + minDistance = Math.min(distance, minDistance); + maxDistance = Math.max(distance, maxDistance); + } else { + distances.push(null); + } + } + } + + // Convert distance data to pixels; + for (let i = 0; i < width * height; i++) { + if (distances[i] === null) { + // Bright white pixel for non-intersections + data.push(255, 255, 255, 255); + } else { + let value = (distances[i] - minDistance) / (maxDistance - minDistance); + value = Math.floor((clamp(value, 0.0, 1.0)) * 255); + data.push(value, value, value, 255); + } + } + + return data; +} diff --git a/test/integration/lib/server.js b/test/integration/lib/server.js index b20eea261fc..a16adc4d2c7 100644 --- a/test/integration/lib/server.js +++ b/test/integration/lib/server.js @@ -20,7 +20,12 @@ module.exports = function () { //Write data to disk const {filePath, data} = JSON.parse(body); - fs.writeFile(path.join(process.cwd(), filePath), data, 'base64', () => { + + let encoding; + if (filePath.split('.')[1] !== 'json') { + encoding = 'base64'; + } + fs.writeFile(path.join(process.cwd(), filePath), data, encoding, () => { res.writeHead(200, {'Content-Type': 'text/html'}); res.end('ok'); }); diff --git a/test/integration/render-tests/combinations/background-translucent--circle-translucent/expected.png b/test/integration/render-tests/combinations/background-translucent--circle-translucent/expected.png index de92554f48f..dcd0daf41af 100644 Binary files a/test/integration/render-tests/combinations/background-translucent--circle-translucent/expected.png and b/test/integration/render-tests/combinations/background-translucent--circle-translucent/expected.png differ diff --git a/test/integration/render-tests/combinations/symbol-translucent--line-translucent-terrain-draped/expected.png b/test/integration/render-tests/combinations/symbol-translucent--line-translucent-terrain-draped/expected.png index 4fbbc3999d3..d17bea7eff6 100644 Binary files a/test/integration/render-tests/combinations/symbol-translucent--line-translucent-terrain-draped/expected.png and b/test/integration/render-tests/combinations/symbol-translucent--line-translucent-terrain-draped/expected.png differ diff --git a/test/integration/render-tests/debug/raster/expected.png b/test/integration/render-tests/debug/raster/expected.png index 04f45dc70c9..e0ed7f8ac69 100644 Binary files a/test/integration/render-tests/debug/raster/expected.png and b/test/integration/render-tests/debug/raster/expected.png differ diff --git a/test/integration/render-tests/debug/raster/style.json b/test/integration/render-tests/debug/raster/style.json index 9cc0d03cf9f..388f9dc4e77 100644 --- a/test/integration/render-tests/debug/raster/style.json +++ b/test/integration/render-tests/debug/raster/style.json @@ -4,7 +4,7 @@ "test": { "debug": true, "height": 256, - "allowed": 0.0062 + "allowed": 0.02975 } }, "center": [ diff --git a/test/integration/render-tests/debug/terrain/collision-lines-occlusion/style.json b/test/integration/render-tests/debug/terrain/collision-lines-occlusion/style.json index c52833756fa..0e9149cf3e1 100644 --- a/test/integration/render-tests/debug/terrain/collision-lines-occlusion/style.json +++ b/test/integration/render-tests/debug/terrain/collision-lines-occlusion/style.json @@ -36,7 +36,7 @@ "minzoom": 6, "maxzoom": 7, "tileSize": 512 - } + } }, "glyphs": "local://glyphs/{fontstack}/{range}.pbf", "layers": [ @@ -67,4 +67,4 @@ "interactive": true } ] -} \ No newline at end of file +} diff --git a/test/integration/render-tests/debug/terrain/collision-occlusion/style.json b/test/integration/render-tests/debug/terrain/collision-occlusion/style.json index 5e083dde5be..2bc0e0b0dc2 100644 --- a/test/integration/render-tests/debug/terrain/collision-occlusion/style.json +++ b/test/integration/render-tests/debug/terrain/collision-occlusion/style.json @@ -8,7 +8,7 @@ "allowed": 0.0005, "description": "Collision debug and occlusion test for text and icons.", "operations": [ - ["wait"] + ["wait", 1500] ] } }, @@ -70,4 +70,4 @@ } } ] -} \ No newline at end of file +} diff --git a/test/integration/render-tests/debug/tile-overscaled/expected.png b/test/integration/render-tests/debug/tile-overscaled/expected.png index 8ea8baa3839..8daae795152 100644 Binary files a/test/integration/render-tests/debug/tile-overscaled/expected.png and b/test/integration/render-tests/debug/tile-overscaled/expected.png differ diff --git a/test/integration/render-tests/debug/tile/expected.png b/test/integration/render-tests/debug/tile/expected.png index 12c0ba92834..fedbdc89213 100644 Binary files a/test/integration/render-tests/debug/tile/expected.png and b/test/integration/render-tests/debug/tile/expected.png differ diff --git a/test/integration/render-tests/fill-extrusion-pattern/multiple-layers-flat/style.json b/test/integration/render-tests/fill-extrusion-pattern/multiple-layers-flat/style.json index b7bcdb52d44..4d91263fa92 100644 --- a/test/integration/render-tests/fill-extrusion-pattern/multiple-layers-flat/style.json +++ b/test/integration/render-tests/fill-extrusion-pattern/multiple-layers-flat/style.json @@ -208,4 +208,4 @@ } } ] -} \ No newline at end of file +} diff --git a/test/integration/render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border/style.json b/test/integration/render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border/style.json index 15d8c0875e3..2044cb9153e 100644 --- a/test/integration/render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border/style.json +++ b/test/integration/render-tests/fill-extrusion-pattern/opacity-terrain-flat-on-border/style.json @@ -167,4 +167,4 @@ } } ] -} \ No newline at end of file +} diff --git a/test/integration/render-tests/fill-extrusion-pattern/tile-buffer/expected.png b/test/integration/render-tests/fill-extrusion-pattern/tile-buffer/expected.png index 8b08b54ea97..5787258e6a0 100644 Binary files a/test/integration/render-tests/fill-extrusion-pattern/tile-buffer/expected.png and b/test/integration/render-tests/fill-extrusion-pattern/tile-buffer/expected.png differ diff --git a/test/integration/render-tests/line-dasharray/case/round/expected.png b/test/integration/render-tests/line-dasharray/case/round/expected.png index 701bdce7e06..310c9429d12 100644 Binary files a/test/integration/render-tests/line-dasharray/case/round/expected.png and b/test/integration/render-tests/line-dasharray/case/round/expected.png differ diff --git a/test/integration/render-tests/regressions/mapbox-gl-js#3682/expected.png b/test/integration/render-tests/regressions/mapbox-gl-js#3682/expected.png deleted file mode 100644 index f5be29d3660..00000000000 Binary files a/test/integration/render-tests/regressions/mapbox-gl-js#3682/expected.png and /dev/null differ diff --git a/test/integration/render-tests/regressions/mapbox-gl-js#3682/style.json b/test/integration/render-tests/regressions/mapbox-gl-js#3682/style.json deleted file mode 100644 index ed5b8bb3ff0..00000000000 --- a/test/integration/render-tests/regressions/mapbox-gl-js#3682/style.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "version": 8, - "metadata": { - "test": { - "width": 64, - "height": 64 - } - }, - "sources": { - "geojson": { - "type": "geojson", - "data": { - "type": "Feature", - "properties": { - "width": 5 - }, - "geometry": { - "type": "LineString", - "coordinates": [ - [ - 0, - -20 - ], - [ - 0, - 20 - ] - ] - } - } - } - }, - "layers": [ - { - "id": "line", - "type": "line", - "source": "geojson", - "paint": { - "line-dasharray": [ - 1, - 1 - ], - "line-width": { - "type": "identity", - "property": "width" - } - } - } - ] -} \ No newline at end of file diff --git a/test/integration/render-tests/runtime-styling/image-update-icon/expected.png b/test/integration/render-tests/runtime-styling/image-update-icon/expected.png index 027baead205..20a306d63c2 100644 Binary files a/test/integration/render-tests/runtime-styling/image-update-icon/expected.png and b/test/integration/render-tests/runtime-styling/image-update-icon/expected.png differ diff --git a/test/integration/render-tests/runtime-styling/image-update-pattern/expected.png b/test/integration/render-tests/runtime-styling/image-update-pattern/expected.png index 0fc06575a26..662b8aebb6c 100644 Binary files a/test/integration/render-tests/runtime-styling/image-update-pattern/expected.png and b/test/integration/render-tests/runtime-styling/image-update-pattern/expected.png differ diff --git a/test/integration/render-tests/video/default/style.json b/test/integration/render-tests/video/default/style.json index 53cca09964f..5e633b67fa9 100644 --- a/test/integration/render-tests/video/default/style.json +++ b/test/integration/render-tests/video/default/style.json @@ -34,7 +34,8 @@ ] ], "urls": [ - "local://video/0.png" + "https://static-assets.mapbox.com/mapbox-gl-js/drone.mp4", + "https://static-assets.mapbox.com/mapbox-gl-js/drone.webm" ] } }, @@ -48,4 +49,4 @@ } } ] -} \ No newline at end of file +} diff --git a/test/integration/rollup.config.test.js b/test/integration/rollup.config.test.js index f8e347cea28..4007dff75ff 100644 --- a/test/integration/rollup.config.test.js +++ b/test/integration/rollup.config.test.js @@ -1,14 +1,16 @@ import {plugins} from '../../build/rollup_plugins'; +const suiteName = process.env.SUITE_NAME; + export default { - input: 'test/integration/lib/query-browser.js', + input: `test/integration/lib/${suiteName}.js`, output: { - name: 'queryTests', + name: `${suiteName}Tests`, format: 'iife', sourcemap: 'inline', indent: false, - file: 'test/integration/dist/query-test.js' + file: `test/integration/dist/integration-test.js` }, - plugins: plugins(false, false), + plugins: plugins(false, false, true), external: [ 'tape', 'mapboxgl' ] }; diff --git a/test/integration/testem.js b/test/integration/testem.js index ec262ff7b0d..bd9601c2ec1 100644 --- a/test/integration/testem.js +++ b/test/integration/testem.js @@ -13,7 +13,10 @@ const rollupDevConfig = require('../../rollup.config').default; const rollupTestConfig = require('./rollup.config.test').default; const rootFixturePath = 'test/integration/'; -const suitePath = 'query-tests'; +const outputPath = `${rootFixturePath}dist`; +const suiteName = process.env.SUITE_NAME; +const suitePath = `${suiteName}-tests`; +const ciOutputFile = `${rootFixturePath}${suitePath}/test-results.xml`; const fixtureBuildInterval = 2000; let beforeHookInvoked = false; @@ -22,21 +25,32 @@ let server; let fixtureWatcher; const rollupWatchers = {}; -module.exports = { - "test_page": "test/integration/testem_page.html", - "src_files": [ - "dist/mapbox-gl-dev.js", - "test/integration/dist/query-test.js" - ], - "launch_in_dev": [], - "launch_in_ci": [ "Chrome" ], - "browser_args": { - "Chrome": { - "mode": "ci", - "args": [ "--headless", "--disable-gpu", "--remote-debugging-port=9222" ] +function getQueryParams() { + const params = process.argv.slice(2).filter((value, index, self) => { return self.indexOf(value) === index; }) || []; + const filterIndex = params.findIndex((elem) => { return String(elem).startsWith("tests="); }); + const queryParams = {}; + if (filterIndex !== -1) { + const split = String(params.splice(filterIndex, 1)).split('='); + if (split.length === 2) { + queryParams.filter = split[1]; } - }, + } + return queryParams; +} + +const defaultTestemConfig = { + "test_page": "test/integration/testem_page.html", + "query_params": getQueryParams(), "proxies": { + "/image":{ + "target": "http://localhost:2900" + }, + "/geojson":{ + "target": "http://localhost:2900" + }, + "/video":{ + "target": "http://localhost:2900" + }, "/tiles":{ "target": "http://localhost:2900" }, @@ -54,6 +68,9 @@ module.exports = { }, "/write-file":{ "target": "http://localhost:2900" + }, + "/mvt-fixtures":{ + "target": "http://localhost:2900" } }, "before_tests"(config, data, callback) { @@ -76,15 +93,31 @@ module.exports = { } }; +const ciTestemConfig = { + "launch_in_ci": [ "Chrome" ], + "reporter": "xunit", + "report_file": ciOutputFile, + "xunit_intermediate_output": true, + "browser_args": { + "Chrome": { + "ci": [ "--disable-backgrounding-occluded-windows" ] + } + } +}; + +const testemConfig = process.env.CI ? Object.assign({}, defaultTestemConfig, ciTestemConfig) : defaultTestemConfig; + +module.exports = testemConfig; + // helper method that builds test artifacts when in CI mode. // Retuns a promise that resolves when all artifacts are built function buildArtifactsCi() { //1. Compile fixture data into a json file, so it can be bundled - generateFixtureJson(rootFixturePath, suitePath); + generateFixtureJson(rootFixturePath, suitePath, outputPath, suitePath === 'render-tests'); //2. Build tape const tapePromise = buildTape(); //3. Build test artifacts in parallel - const rollupPromise = runAll(['build-query-suite', 'build-dev'], {parallel: true}); + const rollupPromise = runAll([`build-test-suite`, 'build-dev'], {parallel: true}); return Promise.all([tapePromise, rollupPromise]); } @@ -95,21 +128,21 @@ function buildArtifactsDev() { return buildTape().then(() => { // A promise that resolves on the first build of fixtures.json return new Promise((resolve, reject) => { - fixtureWatcher = chokidar.watch(getAllFixtureGlobs(rootFixturePath, suitePath)); + fixtureWatcher = chokidar.watch(getAllFixtureGlobs(rootFixturePath, suitePath), {ignored: (path) => path.includes('actual.png') || path.includes('actual.json') || path.includes('diff.png')}); let needsRebuild = false; fixtureWatcher.on('ready', () => { - generateFixtureJson(rootFixturePath, suitePath); + generateFixtureJson(rootFixturePath, suitePath, outputPath, suitePath === 'render-tests'); //Throttle calls to `generateFixtureJson` to run every 2s setInterval(() => { if (needsRebuild) { - generateFixtureJson(rootFixturePath, suitePath); + generateFixtureJson(rootFixturePath, suitePath, outputPath, suitePath === 'render-tests'); needsRebuild = false; } }, fixtureBuildInterval); //Flag needs rebuild when anything changes - fixtureWatcher.on('all', () => { + fixtureWatcher.on('change', () => { needsRebuild = true; }); // Resolve promise once chokidar has finished first scan of fixtures @@ -144,7 +177,7 @@ function buildArtifactsDev() { return Promise.all([ startRollupWatcher('mapbox-gl', rollupDevConfig), - startRollupWatcher('query-suite', rollupTestConfig), + startRollupWatcher(suitePath, rollupTestConfig) ]); }); } diff --git a/test/integration/testem_page.html b/test/integration/testem_page.html index 7cdc390cfa0..0f42781721b 100644 --- a/test/integration/testem_page.html +++ b/test/integration/testem_page.html @@ -3,12 +3,18 @@ Mapbox GL JS Integration Tests + - + - \ No newline at end of file + diff --git a/test/query.test.js b/test/query.test.js deleted file mode 100644 index 2ac5e6584a4..00000000000 --- a/test/query.test.js +++ /dev/null @@ -1,17 +0,0 @@ -/* eslint-disable import/unambiguous, import/no-commonjs, no-global-assign */ - -require('./stub_loader'); -require('@mapbox/flow-remove-types/register'); -require = require("esm")(module, true); - -const querySuite = require('./integration/lib/query'); -const suiteImplementation = require('./suite_implementation'); -const ignores = require('./ignores.json'); - -let tests; - -if (process.argv[1] === __filename && process.argv.length > 2) { - tests = process.argv.slice(2); -} - -querySuite.run('js', {tests, ignores}, suiteImplementation); diff --git a/test/render.test.js b/test/render.test.js deleted file mode 100644 index 3c594a37a71..00000000000 --- a/test/render.test.js +++ /dev/null @@ -1,13 +0,0 @@ -/* eslint-disable import/unambiguous, import/no-commonjs, no-global-assign */ - -require('./stub_loader'); -require('@mapbox/flow-remove-types/register'); -const {registerFont} = require('canvas'); -require = require("esm")(module, true); - -const suite = require('./integration/lib/render'); -const suiteImplementation = require('./suite_implementation'); -const ignores = require('./ignores.json'); -registerFont('./node_modules/npm-font-open-sans/fonts/Bold/OpenSans-Bold.ttf', {family: 'Open Sans', weight: 'bold'}); - -suite.run('js', ignores, suiteImplementation); diff --git a/test/stub_loader.js b/test/stub_loader.js deleted file mode 100644 index 0fc4534a341..00000000000 --- a/test/stub_loader.js +++ /dev/null @@ -1,16 +0,0 @@ -// Load our stubbed ajax module for the integration suite implementation -/* eslint-disable import/unambiguous, import/no-commonjs */ -const fs = require('fs'); -const assert = require('assert'); -const pirates = require('pirates'); - -process.env["ESM_OPTIONS"] = '{ "cache": "node_modules/.cache/esm-stubbed"}'; - -pirates.addHook((code, filename) => { - assert(filename.endsWith('/ajax.js')); - return fs.readFileSync(`${__dirname}/ajax_stubs.js`, 'utf-8'); -}, { - exts: ['.js'], - matcher: filename => filename.endsWith('/ajax.js') -}); - diff --git a/test/util/browser_write_file.js b/test/util/browser_write_file.js index 81de4ca5876..c1297ef0f65 100644 --- a/test/util/browser_write_file.js +++ b/test/util/browser_write_file.js @@ -1,16 +1,20 @@ -/* eslint-disable import/no-commonjs */ /* eslint-env browser */ +/* eslint-disable import/no-commonjs */ +/* global self, WorkerGlobalScope */ // Tests running in the browser need to be able to persist files to disk in certain situations. // our test server (server.js) actually handles the file-io and listens for POST request to /write-file -// This is a helper method to send that POST request. +// This worker provides a helper method to send that POST request. // filepath: relative filepath from the root of the mapboxgl repo // data: base64 encoded string of the data to be persisted to disk -module.exports = function(filepath, data, callback) { +function isWorker() { + return typeof WorkerGlobalScope !== 'undefined' && typeof self !== 'undefined' && self instanceof WorkerGlobalScope; +} +const browserWriteFile = (filepath, data, cb) => { const xhttp = new XMLHttpRequest(); xhttp.onreadystatechange = function() { if (xhttp.readyState === 4 && xhttp.status === 200) { - callback(); + cb(); } }; xhttp.open("POST", "/write-file", true); @@ -22,3 +26,20 @@ module.exports = function(filepath, data, callback) { }; xhttp.send(JSON.stringify(postData)); }; + +if (isWorker()) { + onmessage = function(e) { + e.data.forEach((file) => { + browserWriteFile( + file.path, + file.data, + () => { + self.postMessage(true); + } + ); + }); + }; +} else { + module.exports = browserWriteFile; +} + diff --git a/test/util/html_generator.js b/test/util/html_generator.js new file mode 100644 index 00000000000..86b63af483d --- /dev/null +++ b/test/util/html_generator.js @@ -0,0 +1,131 @@ +/* eslint-env browser */ +import template from 'lodash.template'; + +const CI = process.env.CI; + +const generateResultHTML = template(` +
+ <% if (r.status === 'failed') { %> + + <% } else { %> + + <% } %> + +
+ <% if (r.status !== 'errored') { %> + + <% } %> + <% if (r.expected) { %> + + <% } %> + <% if (r.imgDiff) { %> + + <% } %> + <% if (r.error) { %>

Error: <%- r.error.message %>

<% } %> + <% if (r.jsonDiff) { %> + 
<%- r.jsonDiff.trim() %>
+ <% } %> +
+
+`); + +const pageCss = ` +body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; background: #ecf0f1 } +h1 { font-size: 32px; margin-bottom: 0; } +img { margin: 0 10px 10px 0; border: 1px dotted #ccc; } +input { position: absolute; opacity: 0; z-index: -1;} +.tests { border-top: 1px dotted #bbb; margin-top: 10px; padding-top: 15px; overflow: hidden; } +.status-container { margin: 0; } +.status { text-transform: uppercase; } +.label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; } +.tab { margin-bottom: 30px; width: 100%; overflow: hidden; } +.tab-label { display: flex; color: white; border-radius: 5px; justify-content: space-between; padding: 1em; font-weight: bold; cursor: pointer; } +.tab-label:hover { filter: brightness(85%); } +.tab-label::after { content: "\\276F"; width: 1em; height: 1em; text-align: center; transition: all .35s; } +.tab-content { max-height: 0; padding: 0 1em; background: white; transition: all .35s; } +.tab-content pre { font-size: 14px; margin: 0 0 10px; } +input:checked + .tab-label { filter: brightness(90%); }; +input:checked + .tab-label::after { transform: rotate(90deg); } +input:checked ~ .tab-content { max-height: 100vh; padding: 1em; border: 1px solid #eee; border-top: 0; border-radius: 5px; } +iframe { pointer-events: none; } +`; + +const stats = { + failed: 0, + passed: 0, + todo: 0 +}; +const colors = { + passed: 'green', + failed: 'red', + todo: '#e89b00' +}; + +const counterDom = { + passed: null, + failed: null, + todo: null, +}; + +let resultsContainer; + +export function setupHTML() { + // Add CSS to the page + const style = document.createElement('style'); + document.head.appendChild(style); + style.appendChild(document.createTextNode(pageCss)); + + //Create a container to hold test stats + const statsContainer = document.createElement('div'); + + const failedTestContainer = document.createElement('h1'); + failedTestContainer.style.color = 'red'; + counterDom.failed = document.createElement('span'); + counterDom.failed.innerHTML = '0'; + const failedTests = document.createElement('span'); + failedTests.innerHTML = ' tests failed.'; + failedTestContainer.appendChild(counterDom.failed); + failedTestContainer.appendChild(failedTests); + statsContainer.appendChild(failedTestContainer); + + const passedTestContainer = document.createElement('h1'); + passedTestContainer.style.color = 'green'; + counterDom.passed = document.createElement('span'); + counterDom.passed.innerHTML = '0'; + const passedTests = document.createElement('span'); + passedTests.innerHTML = ' tests passed.'; + passedTestContainer.appendChild(counterDom.passed); + passedTestContainer.appendChild(passedTests); + statsContainer.appendChild(passedTestContainer); + + const todoTestContainer = document.createElement('h1'); + todoTestContainer.style.color = '#e89b00'; + counterDom.todo = document.createElement('span'); + counterDom.todo.innerHTML = '0'; + const todoTests = document.createElement('span'); + todoTests.innerHTML = ' tests todo.'; + todoTestContainer.appendChild(counterDom.todo); + todoTestContainer.appendChild(todoTests); + statsContainer.appendChild(todoTestContainer); + + document.body.appendChild(statsContainer); + + //Create a container to hold test results + resultsContainer = document.createElement('div'); + resultsContainer.className = 'tests'; + document.body.appendChild(resultsContainer); +} + +export function updateHTML(testData) { + const status = testData.status; + stats[status]++; + + testData["color"] = colors[status]; + testData["id"] = `${status}Test-${stats[status]}`; + counterDom[status].innerHTML = stats[status]; + + // skip adding passing tests to report in CI mode + if (CI && status === 'passed') return; + const resultHTMLFrag = document.createRange().createContextualFragment(generateResultHTML({r: testData})); + resultsContainer.appendChild(resultHTMLFrag); +} diff --git a/test/util/tap_html.js b/test/util/tap_html.js deleted file mode 100644 index 560a85406ee..00000000000 --- a/test/util/tap_html.js +++ /dev/null @@ -1,112 +0,0 @@ -/* eslint-disable import/no-commonjs */ -/* eslint-env browser */ -const Parser = require('tap-parser'); -const {template} = require('lodash'); - -const generateResultHTML = template(` -
-

<%- r.status %> <%- r.name %>

- <% if (r.status !== 'errored') { %> - - <% } %> - <% if (r.error) { %>

Error: <%- r.error.message %>

<% } %> - <% if (r.difference) { %> - 
<%- r.difference.trim() %>
- <% } %> -
`); - -const generateStatsHTML = template(` -

-<%- failedTests %> tests failed. -

-

-<%- passedTests %> tests passed. -

-`); - -const pageCss = ` -body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; } -h1 { font-size: 32px; margin-bottom: 0; } -button { vertical-align: middle; } -h2 { font-size: 24px; font-weight: normal; margin: 10px 0 10px; line-height: 1; } -img { margin: 0 10px 10px 0; border: 1px dotted #ccc; } -.stats { margin-top: 10px; } -.test { border-bottom: 1px dotted #bbb; padding-bottom: 5px; } -.tests { border-top: 1px dotted #bbb; margin-top: 10px; } -.diff { color: #777; } -.test p, .test pre { margin: 0 0 10px; } -.test pre { font-size: 14px; } -.label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; } -.hide { display: none; } -`; - -/** - * A class that can be used to incrementally generate prettified HTML output from tap output. - * - * @class TapHtmlGenerator - */ -class TapHtmlGenerator { - - constructor() { - // Add CSS to the page - const style = document.createElement('style'); - document.head.appendChild(style); - style.appendChild(document.createTextNode(pageCss)); - - //Create a container to hold test stats - this.statsContainer = document.createElement('div'); - document.body.appendChild(this.statsContainer); - - //Create a container to hold test results - this.resultsContainer = document.createElement('div'); - this.resultsContainer.className = 'tests'; - document.body.appendChild(this.resultsContainer); - - this.stats = { - failedTests: 0, - passedTests: 0 - }; - - this.tapParser = new Parser(); - this.tapParser.on('pass', this._onTestPassed.bind(this)); - this.tapParser.on('fail', this._onTestFailed.bind(this)); - } - - /** - * Pushes a line of tap output into the html generaor. - * - * @param {string} tapLine - * @memberof TapHtmlGenerator - */ - pushTapLine(tapLine) { - this.tapParser.write(`${tapLine}\n`); - } - - _onTestPassed(assert) { - this.stats.passedTests++; - - const metaData = JSON.parse(assert.name); - metaData["status"] = "passed"; - metaData["color"] = "green"; - - this.resultsContainer.innerHTML += generateResultHTML({r: metaData}); - this._updateStatsContainer(); - } - - _onTestFailed(assert) { - this.stats.failedTests++; - - const metaData = JSON.parse(assert.name); - metaData["status"] = "failed"; - metaData["color"] = "red"; - - this.resultsContainer.innerHTML += generateResultHTML({r: metaData}); - this._updateStatsContainer(); - } - - _updateStatsContainer() { - this.statsContainer.innerHTML = generateStatsHTML(this.stats); - } -} - -module.exports = TapHtmlGenerator; diff --git a/test/util/tape_config.js b/test/util/tape_config.js index ab8e99da491..d640801195d 100644 --- a/test/util/tape_config.js +++ b/test/util/tape_config.js @@ -4,8 +4,7 @@ // This file sets up tape with the add-ons we need, // this file also acts as the entrypoint for browserify. const tape = require('tape'); -const TapHtmlGenerator = require('./tap_html'); -const browserWriteFile = require('./browser_write_file'); +const browserWriteFile = require('../util/browser_write_file.js'); //Add test filtering ability const filter = getQueryVariable('filter') || '.*'; @@ -15,7 +14,8 @@ module.exports = test; // Helper method to extract query params from url function getQueryVariable(variable) { - const query = window.location.search.substring(1); + let query = window.location.search.substring(1); + query = decodeURIComponent(query); const vars = query.split("&"); for (let i = 0; i < vars.length; i++) { const pair = vars[i].split("="); @@ -24,26 +24,20 @@ function getQueryVariable(variable) { return (false); } -const tapHtmlGenerator = new TapHtmlGenerator(); // Testem object is available globally in the browser test page. // Tape outputs via `console.log` and is intercepted by testem using this function Testem.handleConsoleMessage = function(msg) { // Send output over ws to testem server Testem.emit('tap', msg); - // Pipe to html generator - tapHtmlGenerator.pushTapLine(msg); return false; }; // Persist the current html on the page as an artifact once tests finish Testem.afterTests((config, data, cb) => { browserWriteFile( - 'test/integration/query-tests/index.html', + `test/integration/${window._suiteName}/index.html`, window.btoa(document.documentElement.outerHTML), - () => { - cb(); - } + () => cb() ); }); -