From ab231ac97eed808370cd8bd3cd2bd3ed18b90776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sun, 6 Dec 2020 01:41:53 +0100 Subject: [PATCH] Offload generation of blurred backgrounds to a worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now the generation of the blurred backgrounds was done in a normal canvas, so it happened in the main thread and made the UI unresponsive while the image was being generated. Now, to solve that, the blurred background is generated in an OffscreenCanvas in a worker (if supported by the browser). Signed-off-by: Daniel Calviño Sánchez --- lib/Listener/CSPListener.php | 2 + .../CallView/shared/VideoBackground.vue | 4 +- src/utils/imageBlurrer.js | 44 ++++++++++++++++++- src/utils/imageBlurrerWorker.js | 37 ++++++++++++++++ webpack.common.js | 5 +++ 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 src/utils/imageBlurrerWorker.js diff --git a/lib/Listener/CSPListener.php b/lib/Listener/CSPListener.php index 491a0d6fe92..d430a68bb44 100644 --- a/lib/Listener/CSPListener.php +++ b/lib/Listener/CSPListener.php @@ -50,6 +50,8 @@ public function handle(Event $event): void { $csp->addAllowedConnectDomain($server); } + $csp->addAllowedWorkerSrcDomain('\'self\''); + $event->addPolicy($csp); } } diff --git a/src/components/CallView/shared/VideoBackground.vue b/src/components/CallView/shared/VideoBackground.vue index 0db0b28f005..e75325c66c6 100644 --- a/src/components/CallView/shared/VideoBackground.vue +++ b/src/components/CallView/shared/VideoBackground.vue @@ -164,7 +164,9 @@ export default { const image = new Image() image.onload = () => { - this.blurredBackgroundImageSource = image + createImageBitmap(image).then(imageBitmap => { + this.blurredBackgroundImageSource = imageBitmap + }) } image.src = this.backgroundImageUrl }, diff --git a/src/utils/imageBlurrer.js b/src/utils/imageBlurrer.js index d3f2ebce238..8f8705cb3ca 100644 --- a/src/utils/imageBlurrer.js +++ b/src/utils/imageBlurrer.js @@ -19,7 +19,27 @@ * */ -export default function blur(image, width, height, blurRadius) { +import { generateFilePath } from '@nextcloud/router' + +const worker = new Worker(generateFilePath('spreed', '', 'js/image-blurrer-worker.js')) + +const pendingResults = {} +let pendingResultsNextId = 0 + +worker.onmessage = function(message) { + const pendingResult = pendingResults[message.data.id] + if (!pendingResult) { + console.debug('No pending result for blurring image with id ' + message.data.id) + + return + } + + pendingResult(message.data.blurredImageAsDataUrl) + + delete pendingResults[message.data.id] +} + +function blurSync(image, width, height, blurRadius) { return new Promise((resolve, reject) => { const canvas = document.createElement('canvas') canvas.width = width @@ -32,3 +52,25 @@ export default function blur(image, width, height, blurRadius) { resolve(canvas.toDataURL()) }) } + +export default function blur(image, width, height, blurRadius) { + if (typeof OffscreenCanvas === 'undefined') { + return blurSync(image, width, height, blurRadius) + } + + const id = pendingResultsNextId + + pendingResultsNextId++ + + return new Promise((resolve, reject) => { + pendingResults[id] = resolve + + worker.postMessage({ + id: id, + image: image, + width: width, + height: height, + blurRadius: blurRadius, + }) + }) +} diff --git a/src/utils/imageBlurrerWorker.js b/src/utils/imageBlurrerWorker.js new file mode 100644 index 00000000000..756da1ff0a2 --- /dev/null +++ b/src/utils/imageBlurrerWorker.js @@ -0,0 +1,37 @@ +/** + * + * @copyright Copyright (c) 2020, Daniel Calviño Sánchez (danxuliu@gmail.com) + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +const fileReaderSync = new global.FileReaderSync() + +onmessage = function(message) { + const offscreenCanvas = new OffscreenCanvas(message.data.width, message.data.height) + + const context = offscreenCanvas.getContext('2d') + context.filter = `blur(${message.data.blurRadius}px)` + context.drawImage(message.data.image, 0, 0, offscreenCanvas.width, offscreenCanvas.height) + + offscreenCanvas.convertToBlob().then(blob => { + postMessage({ + id: message.data.id, + blurredImageAsDataUrl: fileReaderSync.readAsDataURL(blob), + }) + }) +} diff --git a/webpack.common.js b/webpack.common.js index 42f733aa7da..8fd83749819 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -7,6 +7,11 @@ module.exports = { entry: { 'admin-settings': path.join(__dirname, 'src', 'mainAdminSettings.js'), 'collections': path.join(__dirname, 'src', 'collections.js'), + // There is a "worker-loader" plugin for Webpack, but I was not able to + // get it to work ("publicPath" uses "output.publicPath" rather than the + // one set in the plugin + // https://github.com/webpack-contrib/worker-loader/issues/281). + 'image-blurrer-worker': path.join(__dirname, 'src', 'utils/imageBlurrerWorker.js'), 'talk': path.join(__dirname, 'src', 'main.js'), 'talk-files-sidebar': [ path.join(__dirname, 'src', 'mainFilesSidebar.js'),