Progressive enhancement for decoding DEM data on workers with ImageBi…

…tmap and OffscreenCanvas (#8845)

bench/benchmarks/hillshade_load.js

@@ -0,0 +1,50 @@
+// @flow
+
+import Benchmark from '../lib/benchmark';
+import createMap from '../lib/create_map';
+import type {StyleSpecification} from '../../src/style-spec/types';
+
+export default class HillshadeLoad extends Benchmark {
+    style: StyleSpecification;
+
+    constructor() {
+        super();
+        this.style = {
+            "version": 8,
+            "name": "Hillshade-only",
+            "center": [-112.81596278901452, 37.251160384573595],
+            "zoom": 11.560975632435424,
+            "bearing": 0,
+            "pitch": 0,
+            "sources": {
+                "mapbox://mapbox.terrain-rgb": {
+                    "url": "mapbox://mapbox.terrain-rgb",
+                    "type": "raster-dem",
+                    "tileSize": 256
+                }
+            },
+            "layers": [
+                {
+                    "id": "mapbox-terrain-rgb",
+                    "type": "hillshade",
+                    "source": "mapbox://mapbox.terrain-rgb",
+                    "layout": {},
+                    "paint": {}
+                }
+            ]
+        };
+    }
+
+    bench() {
+        return createMap({
+            width: 1024,
+            height: 1024,
+            style: this.style,
+            stubRender: false,
+            showMap: true,
+            idle: true
+        }).then((map) => {
+            map.remove();
+        });
+    }
+}

bench/lib/create_map.js

@@ -4,12 +4,20 @@ import Map from '../../src/ui/map';
 
 export default function (options: any): Promise<Map> {
     return new Promise((resolve, reject) => {
+        if (options) {
+            options.stubRender = options.stubRender == null ? true : options.stubRender;
+            options.showMap = options.showMap == null ? false : options.showMap;
+        }
+
         const container = document.createElement('div');
         container.style.width = `${options.width || 512}px`;
         container.style.height = `${options.height || 512}px`;
         container.style.margin = '0 auto';
         container.style.display = 'block';
-        container.style.visibility = 'hidden';
+
+        if (!options.showMap) {
+            container.style.visibility = 'hidden';
+        }
         (document.body: any).appendChild(container);
 
         const map = new Map(Object.assign({
@@ -19,15 +27,16 @@ export default function (options: any): Promise<Map> {
 
         map
             .on(options.idle ? 'idle' : 'load', () => {
-                // Stub out `_rerender`; benchmarks need to be the only trigger of `_render` from here on out.
-                map._rerender = () => {};
+                if (options.stubRender) {
+                    // Stub out `_rerender`; benchmarks need to be the only trigger of `_render` from here on out.
+                    map._rerender = () => {};
 
-                // If there's a pending rerender, cancel it.
-                if (map._frame) {
-                    map._frame.cancel();
-                    map._frame = null;
+                    // If there's a pending rerender, cancel it.
+                    if (map._frame) {
+                        map._frame.cancel();
+                        map._frame = null;
+                    }
                 }
-
                 resolve(map);
             })
             .on('error', (e) => reject(e.error))

bench/versions/benchmarks.js

@@ -12,6 +12,7 @@ import PaintStates from '../benchmarks/paint_states';
 import {PropertyLevelRemove, FeatureLevelRemove, SourceLevelRemove} from '../benchmarks/remove_paint_state';
 import {LayerBackground, LayerCircle, LayerFill, LayerFillExtrusion, LayerHeatmap, LayerHillshade, LayerLine, LayerRaster, LayerSymbol, LayerSymbolWithIcons} from '../benchmarks/layers';
 import Load from '../benchmarks/map_load';
+import HillshadeLoad from '../benchmarks/hillshade_load';
 import Validate from '../benchmarks/style_validate';
 import StyleLayerCreate from '../benchmarks/style_layer_create';
 import QueryPoint from '../benchmarks/query_point';
@@ -71,6 +72,7 @@ register('LayoutDDS', new LayoutDDS());
 register('SymbolLayout', new SymbolLayout(style, styleLocations.map(location => location.tileID[0])));
 register('FilterCreate', new FilterCreate());
 register('FilterEvaluate', new FilterEvaluate());
+register('HillshadeLoad', new HillshadeLoad());
 
 Promise.resolve().then(() => {
     // Ensure the global worker pool is never drained. Browsers have resource limits

flow-typed/offscreen-canvas.js

@@ -0,0 +1,9 @@
+// @flow strict
+
+declare class OffscreenCanvas {
+    width: number;
+    height: number;
+
+    constructor(width: number, height: number): OffscreenCanvas;
+    getContext(contextType: '2d'): CanvasRenderingContext2D;
+}

src/source/image_source.js

@@ -78,7 +78,7 @@ class ImageSource extends Evented implements Source {
     dispatcher: Dispatcher;
     map: Map;
     texture: Texture | null;
-    image: HTMLImageElement;
+    image: HTMLImageElement | ImageBitmap;
     tileID: CanonicalTileID;
     _boundsArray: RasterBoundsArray;
     boundsBuffer: VertexBuffer;

src/source/raster_dem_tile_source.js

@@ -4,6 +4,8 @@ import {getImage, ResourceType} from '../util/ajax';
 import {extend} from '../util/util';
 import {Evented} from '../util/evented';
 import browser from '../util/browser';
+import window from '../util/window';
+import offscreenCanvasSupported from '../util/offscreen_canvas_supported';
 import {OverscaledTileID} from './tile_id';
 import RasterTileSource from './raster_tile_source';
 // ensure DEMData is registered for worker transfer on main thread:
@@ -54,7 +56,8 @@ class RasterDEMTileSource extends RasterTileSource implements Source {
                 if (this.map._refreshExpiredTiles) tile.setExpiryData(img);
                 delete (img: any).cacheControl;
                 delete (img: any).expires;
-                const rawImageData = browser.getImageData(img, 1);
+                const transfer = window.ImageBitmap && img instanceof window.ImageBitmap && offscreenCanvasSupported();
+                const rawImageData = transfer ? img : browser.getImageData(img, 1);
                 const params = {
                     uid: tile.uid,
                     coord: tile.tileID,

src/source/raster_dem_tile_worker_source.js

@@ -1,31 +1,55 @@
 // @flow
 
 import DEMData from '../data/dem_data';
+import {RGBAImage} from '../util/image';
+import window from '../util/window';
 
 import type Actor from '../util/actor';
 import type {
     WorkerDEMTileParameters,
     WorkerDEMTileCallback,
     TileParameters
 } from './worker_source';
+const {ImageBitmap} = window;
 
 class RasterDEMTileWorkerSource {
     actor: Actor;
     loaded: {[string]: DEMData};
+    offscreenCanvas: OffscreenCanvas;
+    offscreenCanvasContext: CanvasRenderingContext2D;
 
     constructor() {
         this.loaded = {};
     }
 
     loadTile(params: WorkerDEMTileParameters, callback: WorkerDEMTileCallback) {
         const {uid, encoding, rawImageData} = params;
-        const dem = new DEMData(uid, rawImageData, encoding);
-
+        // Main thread will transfer ImageBitmap if offscreen decode with OffscreenCanvas is supported, else it will transfer an already decoded image.
+        const imagePixels = (ImageBitmap && rawImageData instanceof ImageBitmap) ? this.getImageData(rawImageData) : rawImageData;
+        const dem = new DEMData(uid, imagePixels, encoding);
         this.loaded = this.loaded || {};
         this.loaded[uid] = dem;
         callback(null, dem);
     }
 
+    getImageData(imgBitmap: ImageBitmap): RGBAImage {
+        // Lazily initialize OffscreenCanvas
+        if (!this.offscreenCanvas || !this.offscreenCanvasContext) {
+            // Dem tiles are typically 256x256
+            this.offscreenCanvas = new OffscreenCanvas(imgBitmap.width, imgBitmap.height);
+            this.offscreenCanvasContext = this.offscreenCanvas.getContext('2d');
+        }
+
+        this.offscreenCanvas.width = imgBitmap.width;
+        this.offscreenCanvas.height = imgBitmap.height;
+
+        this.offscreenCanvasContext.drawImage(imgBitmap, 0, 0, imgBitmap.width, imgBitmap.height);
+        // Insert an additional 1px padding around the image to allow backfilling for neighboring data.
+        const imgData = this.offscreenCanvasContext.getImageData(-1, -1, imgBitmap.width + 2, imgBitmap.height + 2);
+        this.offscreenCanvasContext.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
+        return new RGBAImage({width: imgData.width, height: imgData.height}, imgData.data);
+    }
+
     removeTile(params: TileParameters) {
         const loaded = this.loaded,
             uid = params.uid;

src/source/worker_source.js

@@ -12,6 +12,8 @@ import type DEMData from '../data/dem_data';
 import type {StyleGlyph} from '../style/style_glyph';
 import type {StyleImage} from '../style/style_image';
 import type {PromoteIdSpecification} from '../style-spec/types';
+import window from '../util/window';
+const {ImageBitmap} = window;
 
 export type TileParameters = {
     source: string,
@@ -33,7 +35,7 @@ export type WorkerTileParameters = TileParameters & {
 
 export type WorkerDEMTileParameters = TileParameters & {
     coord: { z: number, x: number, y: number, w: number },
-    rawImageData: RGBAImage,
+    rawImageData: RGBAImage | ImageBitmap,
     encoding: "mapbox" | "terrarium"
 };
 

src/util/ajax.js

@@ -7,6 +7,7 @@ import config from './config';
 import assert from 'assert';
 import {cacheGet, cachePut} from './tile_request_cache';
 import webpSupported from './webp_supported';
+import offscreenCanvasSupported from './offscreen_canvas_supported';
 
 import type {Callback} from '../types/callback';
 import type {Cancelable} from '../types/cancelable';
@@ -257,14 +258,37 @@ function sameOrigin(url) {
 
 const transparentPngUrl = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVQYV2NgAAIAAAUAAarVyFEAAAAASUVORK5CYII=';
 
+function arrayBufferToImage(data: ArrayBuffer, callback: (err: ?Error, image: ?HTMLImageElement) => void, cacheControl: ?string, expires: ?string) {
+    const img: HTMLImageElement = new window.Image();
+    const URL = window.URL;
+    img.onload = () => {
+        callback(null, img);
+        URL.revokeObjectURL(img.src);
+    };
+    img.onerror = () => callback(new Error('Could not load image. Please make sure to use a supported image type such as PNG or JPEG. Note that SVGs are not supported.'));
+    const blob: Blob = new window.Blob([new Uint8Array(data)], {type: 'image/png'});
+    (img: any).cacheControl = cacheControl;
+    (img: any).expires = expires;
+    img.src = data.byteLength ? URL.createObjectURL(blob) : transparentPngUrl;
+}
+
+function arrayBufferToImageBitmap(data: ArrayBuffer, callback: (err: ?Error, image: ?ImageBitmap) => void) {
+    const blob: Blob = new window.Blob([new Uint8Array(data)], {type: 'image/png'});
+    window.createImageBitmap(blob).then((imgBitmap) => {
+        callback(null, imgBitmap);
+    }).catch(() => {
+        callback(new Error('Could not load image. Please make sure to use a supported image type such as PNG or JPEG. Note that SVGs are not supported.'));
+    });
+}
+
 let imageQueue, numImageRequests;
 export const resetImageRequestQueue = () => {
     imageQueue = [];
     numImageRequests = 0;
 };
 resetImageRequestQueue();
 
-export const getImage = function(requestParameters: RequestParameters, callback: Callback<HTMLImageElement>): Cancelable {
+export const getImage = function(requestParameters: RequestParameters, callback: Callback<HTMLImageElement | ImageBitmap>): Cancelable {
     if (webpSupported.supported) {
         if (!requestParameters.headers) {
             requestParameters.headers = {};
@@ -309,16 +333,11 @@ export const getImage = function(requestParameters: RequestParameters, callback:
         if (err) {
             callback(err);
         } else if (data) {
-            const img: HTMLImageElement = new window.Image();
-            img.onload = () => {
-                callback(null, img);
-                window.URL.revokeObjectURL(img.src);
-            };
-            img.onerror = () => callback(new Error('Could not load image. Please make sure to use a supported image type such as PNG or JPEG. Note that SVGs are not supported.'));
-            const blob: Blob = new window.Blob([new Uint8Array(data)], {type: 'image/png'});
-            (img: any).cacheControl = cacheControl;
-            (img: any).expires = expires;
-            img.src = data.byteLength ? window.URL.createObjectURL(blob) : transparentPngUrl;
+            if (offscreenCanvasSupported()) {
+                arrayBufferToImageBitmap(data, callback);
+            } else {
+                arrayBufferToImage(data, callback, cacheControl, expires);
+            }
         }
     });
 

src/util/offscreen_canvas_supported.js

@@ -0,0 +1,14 @@
+// @flow
+import window from './window';
+
+let supportsOffscreenCanvas: ?boolean;
+
+export default function offscreenCanvasSupported(): boolean {
+    if (supportsOffscreenCanvas == null) {
+        supportsOffscreenCanvas = window.OffscreenCanvas &&
+            new window.OffscreenCanvas(1, 1).getContext('2d') &&
+            typeof window.createImageBitmap === 'function';
+    }
+
+    return supportsOffscreenCanvas;
+}

src/util/web_worker_transfer.js

@@ -9,7 +9,7 @@ import CompoundExpression from '../style-spec/expression/compound_expression';
 import expressions from '../style-spec/expression/definitions';
 import ResolvedImage from '../style-spec/expression/types/resolved_image';
 import window from './window';
-const {ImageData} = window;
+const {ImageData, ImageBitmap} = window;
 
 import type {Transferable} from '../types/transferable';
 
@@ -105,6 +105,11 @@ function isArrayBuffer(val: any): boolean {
            (val instanceof ArrayBuffer || (val.constructor && val.constructor.name === 'ArrayBuffer'));
 }
 
+function isImageBitmap(val: any): boolean {
+    return ImageBitmap &&
+        val instanceof ImageBitmap;
+}
+
 /**
  * Serialize the given object for transfer to or from a web worker.
  *
@@ -133,7 +138,7 @@ export function serialize(input: mixed, transferables: ?Array<Transferable>): Se
         return input;
     }
 
-    if (isArrayBuffer(input)) {
+    if (isArrayBuffer(input) || isImageBitmap(input)) {
         if (transferables) {
             transferables.push(((input: any): ArrayBuffer));
         }
@@ -224,6 +229,7 @@ export function deserialize(input: Serialized): mixed {
         input instanceof Date ||
         input instanceof RegExp ||
         isArrayBuffer(input) ||
+        isImageBitmap(input) ||
         ArrayBuffer.isView(input) ||
         input instanceof ImageData) {
         return input;