From 2b7c145d2a4461d618052f8f908f1f47ceffb7f2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 9 Dec 2019 15:57:51 +0100 Subject: [PATCH] feat: preload DNSLink websites in background With https://github.com/ipfs-shipyard/ipfs-companion/pull/826 data was no longer getting into the cache of local node. This adds background queue that ensures every visited DNSLink asset is preloaded to the cache of local IPFS node. --- add-on/_locales/en/messages.json | 8 ++++ add-on/src/lib/dnslink.js | 49 ++++++++++++++++-------- add-on/src/lib/ipfs-request.js | 19 +++++---- add-on/src/lib/options.js | 1 + add-on/src/options/forms/dnslink-form.js | 11 ++++++ add-on/src/options/page.js | 1 + 6 files changed, 65 insertions(+), 24 deletions(-) diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index e762d3bb2..50b031b3d 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -287,6 +287,14 @@ "message": "If global redirect is enabled, this will include DNSLink websites and redirect them to respective /ipns/{fqdn} paths at Custom Gateway", "description": "An option description on the Preferences screen (option_dnslinkRedirect_description)" }, + "option_dnslinkDataPreload_title": { + "message": "Preload visited pages", + "description": "An option title on the Preferences screen (option_dnslinkDataPreload_title)" + }, + "option_dnslinkDataPreload_description": { + "message": "DNSLink websites will be preloaded to the local IPFS node to enable offline access and improve resiliency against network failures", + "description": "An option description on the Preferences screen (option_dnslinkDataPreload_description)" + }, "option_dnslinkRedirect_warning": { "message": "Redirecting to a path-based gateway breaks Origin-based security isolation of DNSLink website! Please leave this disabled unless you are aware of (and ok with) related risks.", "description": "A warning on the Preferences screen, displayed when URL does not belong to Secure Context (option_customGatewayUrl_warning)" diff --git a/add-on/src/lib/dnslink.js b/add-on/src/lib/dnslink.js index 6414b01d0..891fb9050 100644 --- a/add-on/src/lib/dnslink.js +++ b/add-on/src/lib/dnslink.js @@ -17,8 +17,11 @@ module.exports = function createDnslinkResolver (getState) { // DNSLink lookup result cache const cacheOptions = { max: 1000, maxAge: 1000 * 60 * 60 * 12 } const cache = new LRU(cacheOptions) - // upper bound for concurrent background lookups done by preloadDnslink(url) - const lookupQueue = new PQueue({ concurrency: 8 }) + // upper bound for concurrent background lookups done by resolve(url) + const lookupQueue = new PQueue({ concurrency: 4 }) + // preload of DNSLink data + const preloadUrlCache = new LRU(cacheOptions) + const preloadQueue = new PQueue({ concurrency: 4 }) const dnslinkResolver = { @@ -89,20 +92,34 @@ module.exports = function createDnslinkResolver (getState) { return dnslink }, - // does not return anything, runs async lookup in the background - // and saves result into cache with an optional callback - preloadDnslink (url, cb) { - if (dnslinkResolver.canLookupURL(url)) { - lookupQueue.add(async () => { - const fqdn = new URL(url).hostname - const result = dnslinkResolver.readAndCacheDnslink(fqdn) - if (cb) { - cb(result) - } - }) - } else if (cb) { - cb(null) - } + // runs async lookup in a queue in the background and returns the record + async resolve (url) { + if (!dnslinkResolver.canLookupURL(url)) return + const fqdn = new URL(url).hostname + const cachedResult = dnslinkResolver.cachedDnslink(fqdn) + if (cachedResult) return cachedResult + return lookupQueue.add(() => { + return dnslinkResolver.readAndCacheDnslink(fqdn) + }) + }, + + // preloads data behind the url to local node + async preloadData (url) { + const state = getState() + if (!state.dnslinkDataPreload) return + if (preloadUrlCache.get(url)) return + preloadUrlCache.set(url, true) + const dnslink = await dnslinkResolver.resolve(url) + if (!dnslink) return + if (state.ipfsNodeType === 'embedded') return + if (state.peerCount < 1) return + return preloadQueue.add(async () => { + const { pathname } = new URL(url) + const preloadUrl = new URL(state.gwURLString) + preloadUrl.pathname = `${dnslink}${pathname}` + await fetch(preloadUrl.toString(), { method: 'HEAD' }) + return preloadUrl + }) }, // low level lookup without cache diff --git a/add-on/src/lib/ipfs-request.js b/add-on/src/lib/ipfs-request.js index 76fb317f3..d0e870ef2 100644 --- a/add-on/src/lib/ipfs-request.js +++ b/add-on/src/lib/ipfs-request.js @@ -89,7 +89,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru if (request.type === 'main_frame') { // lazily trigger DNSLink lookup (will do anything only if status for root domain is not in cache) if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) { - dnslinkResolver.preloadDnslink(request.url) + dnslinkResolver.resolve(request.url) // no await: preload record in background } } return isIgnored(request.requestId) @@ -142,15 +142,18 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru return redirectToGateway(request.url, state, ipfsPathValidator) } // Detect dnslink using heuristics enabled in Preferences - if (state.dnslinkRedirect && state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) { - const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url) - if (dnslinkRedirect && isSafeToRedirect(request, runtime)) { - // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect) - return dnslinkRedirect + if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) { + if (state.dnslinkRedirect) { + const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url) + if (dnslinkRedirect && isSafeToRedirect(request, runtime)) { + // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect) + return dnslinkRedirect + } + } else if (state.dnslinkDataPreload) { + dnslinkResolver.preloadData(request.url) } if (state.dnslinkPolicy === 'best-effort') { - // dnslinkResolver.preloadDnslink(request.url, (dnslink) => console.log(`---> preloadDnslink(${new URL(request.url).hostname})=${dnslink}`)) - dnslinkResolver.preloadDnslink(request.url) + dnslinkResolver.resolve(request.url) } } } diff --git a/add-on/src/lib/options.js b/add-on/src/lib/options.js index 3dc5dad41..0805f39bf 100644 --- a/add-on/src/lib/options.js +++ b/add-on/src/lib/options.js @@ -17,6 +17,7 @@ exports.optionDefaults = Object.freeze({ automaticMode: true, linkify: false, dnslinkPolicy: 'best-effort', + dnslinkDataPreload: true, dnslinkRedirect: false, recoverFailedHttpRequests: true, detectIpfsPathHeader: true, diff --git a/add-on/src/options/forms/dnslink-form.js b/add-on/src/options/forms/dnslink-form.js index dfbf77399..03d03125b 100644 --- a/add-on/src/options/forms/dnslink-form.js +++ b/add-on/src/options/forms/dnslink-form.js @@ -7,11 +7,13 @@ const switchToggle = require('../../pages/components/switch-toggle') function dnslinkForm ({ dnslinkPolicy, + dnslinkDataPreload, dnslinkRedirect, onOptionChange }) { const onDnslinkPolicyChange = onOptionChange('dnslinkPolicy') const onDnslinkRedirectChange = onOptionChange('dnslinkRedirect') + const onDnslinkDataPreloadChange = onOptionChange('dnslinkDataPreload') return html`
@@ -47,6 +49,15 @@ function dnslinkForm ({ +
+ +
${switchToggle({ id: 'dnslinkDataPreload', checked: dnslinkDataPreload, onchange: onDnslinkDataPreloadChange })}
+