feat: recover from DNS failures, support DNSLink for .eth

add-on/src/lib/dnslink.js

@@ -180,8 +180,7 @@ module.exports = function createDnslinkResolver (getState) {
       if (typeof url === 'string') {
         url = new URL(url)
       }
-      const fqdn = url.hostname
-      return `/ipns/${fqdn}${url.pathname}${url.search}${url.hash}`
+      return `/ipns/${url.hostname}${url.pathname}${url.search}${url.hash}`
     },
 
     // Test if URL contains a valid DNSLink FQDN

add-on/src/lib/ipfs-request.js

@@ -9,36 +9,20 @@ const LRU = require('lru-cache')
 const IsIpfs = require('is-ipfs')
 const isFQDN = require('is-fqdn')
 const { pathAtHttpGateway } = require('./ipfs-path')
+
 const redirectOptOutHint = 'x-ipfs-companion-no-redirect'
-const recoverableErrors = new Set([
+const recoverableNetworkErrors = new Set([
   // Firefox
+  'NS_ERROR_UNKNOWN_HOST', // dns failure
   'NS_ERROR_NET_TIMEOUT', // eg. httpd is offline
   'NS_ERROR_NET_RESET', // failed to load because the server kept reseting the connection
   'NS_ERROR_NET_ON_RESOLVED', // no network
   // Chrome
+  'net::ERR_NAME_NOT_RESOLVED', // dns failure
   'net::ERR_CONNECTION_TIMED_OUT', // eg. httpd is offline
   'net::ERR_INTERNET_DISCONNECTED' // no network
 ])
-
-const recoverableErrorCodes = new Set([
-  404,
-  408,
-  410,
-  415,
-  451,
-  500,
-  502,
-  503,
-  504,
-  509,
-  520,
-  521,
-  522,
-  523,
-  524,
-  525,
-  526
-])
+const recoverableHttpError = (code) => code && code >= 400
 
 // Request modifier provides event listeners for the various stages of making an HTTP request
 // API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
@@ -171,11 +155,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     // This is a good place to listen if you want to modify HTTP request headers.
     onBeforeSendHeaders (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive
-      if (!state.active) {
-        return
-      }
+      if (!state.active) return
 
       // Special handling of requests made to API
       if (request.url.startsWith(state.apiURLString)) {
@@ -286,11 +266,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     // You can use this event to modify HTTP response headers or do a very late redirect.
     onHeadersReceived (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive
-      if (!state.active) {
-        return
-      }
+      if (!state.active) return
 
       // Special handling of requests made to API
       if (request.url.startsWith(state.apiURLString)) {
@@ -387,58 +363,53 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     },
 
     // browser.webRequest.onErrorOccurred
-    // Fired when a request could not be processed due to an error:
-    // for example, a lack of Internet connectivity.
+    // Fired when a request could not be processed due to an error on network level.
+    // For example: TCP timeout, DNS lookup failure
     async onErrorOccurred (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive or request is marked as ignored
-      if (!state.active || isIgnored(request.requestId)) {
-        return
+      if (!state.active) return
+
+      // Check if error can be recovered via EthDNS
+      if (isRecoverableViaEthDNS(request, state)) {
+        const url = new URL(request.url)
+        url.hostname = `${url.hostname}.link`
+        const redirect = { redirectUrl: url.toString() }
+        log(`onErrorOccurred: attempting to recover from DNS error (${request.error}) using EthDNS for ${request.url}`, redirect.redirectUrl)
+        return createTabWithURL(redirect, browser)
       }
 
-      // console.log('onErrorOccurred:' + request.error)
-      // console.log('onErrorOccurred', request)
-      // Check if error is final and can be recovered via DNSLink
-      let redirect
-      const recoverableViaDnslink =
-        state.dnslinkPolicy &&
-        request.type === 'main_frame' &&
-        recoverableErrors.has(request.error)
-      if (recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)) {
-        // Explicit call to ignore global DNSLink policy and force DNS TXT lookup
-        const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
-        redirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
-        log(`onErrorOccurred: attempting to recover using dnslink for ${request.url}`, redirect)
+      // Check if error can be recovered via DNSLink
+      if (isRecoverableViaDNSLink(request, state, dnslinkResolver)) {
+        const { hostname } = new URL(request.url)
+        const dnslink = dnslinkResolver.readAndCacheDnslink(hostname)
+        if (dnslink) {
+          const redirect = dnslinkResolver.dnslinkRedirect(request.url, dnslink)
+          log(`onErrorOccurred: attempting to recover from network error (${request.error}) using dnslink for ${request.url}`, redirect.redirectUrl)
+          return createTabWithURL(redirect, browser)
+        }
       }
-      // if error cannot be recovered via DNSLink
-      // direct the request to the public gateway
-      const recoverable = isRecoverable(request, state, ipfsPathValidator)
-      if (!redirect && recoverable) {
+
+      // Check if error can be recovered by opening same content-addresed path
+      // using active gateway (public or local, depending on redirect state)
+      if (isRecoverable(request, state, ipfsPathValidator)) {
         const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
-        redirect = { redirectUrl }
-        log(`onErrorOccurred: attempting to recover failed request for ${request.url}`, redirect)
-      }
-      // We can't redirect in onErrorOccurred, so if DNSLink is present
-      // recover by opening IPNS version in a new tab
-      // TODO: add tests and demo
-      if (redirect) {
-        createTabWithURL(redirect, browser)
+        log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl)
+        return createTabWithURL({ redirectUrl }, browser)
       }
     },
 
+    // browser.webRequest.onCompleted
+    // Fired when HTTP request is completed (successfully or with an error code)
     async onCompleted (request) {
       const state = getState()
-
-      const recoverable =
-        isRecoverable(request, state, ipfsPathValidator) &&
-        recoverableErrorCodes.has(request.statusCode)
-      if (recoverable) {
+      if (!state.active) return
+      if (request.statusCode === 200) return // finish if no error to recover from
+      if (isRecoverable(request, state, ipfsPathValidator)) {
         const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
         const redirect = { redirectUrl }
         if (redirect) {
-          log(`onCompleted: attempting to recover failed request for ${request.url}`, redirect)
-          createTabWithURL(redirect, browser)
+          log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect)
+          return createTabWithURL(redirect, browser)
         }
       }
     }
@@ -548,18 +519,41 @@ function findHeaderIndex (name, headers) {
   return headers.findIndex(x => x.name && x.name.toLowerCase() === name.toLowerCase())
 }
 
-// utility functions for handling redirects
-// from onErrorOccurred and onCompleted
+// RECOVERY OF FAILED REQUESTS
+// ===================================================================
+
+// Recovery check for onErrorOccurred (request.error) and onCompleted (request.statusCode)
 function isRecoverable (request, state, ipfsPathValidator) {
   return state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) &&
     ipfsPathValidator.publicIpfsOrIpnsResource(request.url) &&
-    !request.url.startsWith(state.pubGwURLString) &&
-    request.type === 'main_frame'
+    !request.url.startsWith(state.pubGwURLString)
+}
+
+// Recovery check for onErrorOccurred (request.error)
+function isRecoverableViaDNSLink (request, state, dnslinkResolver) {
+  const recoverableViaDnslink =
+    state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    state.dnslinkPolicy &&
+    recoverableNetworkErrors.has(request.error)
+  return recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)
+}
+
+// Recovery check for onErrorOccurred (request.error)
+function isRecoverableViaEthDNS (request, state) {
+  return state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    recoverableNetworkErrors.has(request.error) &&
+    new URL(request.url).hostname.endsWith('.eth')
 }
 
+// We can't redirect in onErrorOccurred/onCompleted
+// Indead, we recover by opening URL in a new tab that replaces the failed one
 async function createTabWithURL (redirect, browser) {
   const currentTabId = await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0].id)
-  await browser.tabs.create({
+  return browser.tabs.create({
     active: true,
     openerTabId: currentTabId,
     url: redirect.redirectUrl

add-on/src/options/forms/experiments-form.js

@@ -59,6 +59,15 @@ function experimentsForm ({
           </label>
           <div>${switchToggle({ id: 'catchUnhandledProtocols', checked: catchUnhandledProtocols, onchange: onCatchUnhandledProtocolsChange })}</div>
         </div>
+        <div>
+          <label for="recoverFailedHttpRequests">
+            <dl>
+              <dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
+              <dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
+            </dl>
+          </label>
+          <div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
+        </div>
         <div>
           <label for="linkify">
             <dl>
@@ -98,15 +107,6 @@ function experimentsForm ({
             </option>
           </select>
         </div>
-        <div>
-          <label for="recoverFailedHttpRequests">
-            <dl>
-              <dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
-              <dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
-            </dl>
-          </label>
-          <div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
-        </div>
         <div>
           <label for="detectIpfsPathHeader">
             <dl>

test/functional/lib/ipfs-request-gateway-recover.test.js

@@ -18,7 +18,11 @@ const urlRequestWithStatus = (url, statusCode = 200, type = 'main_frame') => {
   return { ...url2request(url, type), statusCode }
 }
 
-describe('requestHandler.onCompleted:', function () {
+const urlRequestWithNetworkError = (url, error = 'net::ERR_CONNECTION_TIMED_OUT', type = 'main_frame') => {
+  return { ...url2request(url, type), error }
+}
+
+describe('requestHandler.onCompleted:', function () { // HTTP-level errors
   let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
 
   before(function () {
@@ -40,6 +44,7 @@ describe('requestHandler.onCompleted:', function () {
   describe('with recoverFailedHttpRequests=true', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = true
+      state.dnslinkPolicy = false
     })
     it('should do nothing if broken request is a non-IPFS request', async function () {
       const request = urlRequestWithStatus('https://wikipedia.org', 500)
@@ -61,7 +66,7 @@ describe('requestHandler.onCompleted:', function () {
       await requestHandler.onCompleted(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
-    it('should redirect broken non-default public gateway IPFS request to public gateway', async function () {
+    it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
       const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
       await requestHandler.onCompleted(request)
       assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
@@ -71,6 +76,7 @@ describe('requestHandler.onCompleted:', function () {
   describe('with recoverFailedHttpRequests=false', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = false
+      state.dnslinkPolicy = false
     })
     it('should do nothing on broken non-default public gateway IPFS request', async function () {
       const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
@@ -90,7 +96,7 @@ describe('requestHandler.onCompleted:', function () {
   })
 })
 
-describe('requestHandler.onErrorOccurred:', function () {
+describe('requestHandler.onErrorOccurred:', function () { // network errors
   let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
 
   before(function () {
@@ -112,42 +118,83 @@ describe('requestHandler.onErrorOccurred:', function () {
   describe('with recoverFailedHttpRequests=true', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = true
+      state.dnslinkPolicy = false
     })
     it('should do nothing if failed request is a non-IPFS request', async function () {
-      const request = url2request('https://wikipedia.org', 500)
+      const request = urlRequestWithNetworkError('https://wikipedia.org')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is a non-public IPFS request', async function () {
-      const request = url2request('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
+      const request = urlRequestWithNetworkError('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is to the default public gateway', async function () {
-      const request = url2request('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
+      const request = urlRequestWithNetworkError('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is not a \'main_frame\' request', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'stylesheet')
+      const requestType = 'stylesheet'
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'net::ERR_NAME_NOT_RESOLVED', requestType)
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
-    it('should redirect failed non-default public gateway IPFS request to public gateway', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+    it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
     })
+    it('should recover from unreachable HTTP server by reopening DNSLink on the public gateway', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
+      const expectedUrl = 'http://127.0.0.1:8080/ipns/en.wikipedia-on-ipfs.org/'
+      const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org/')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
+      dnslinkResolver.clearCache()
+    })
+    it('should recover from failed DNS for .eth opening it on EthDNS gateway at .eth.link', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('almonit.eth', false)
+      dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
+      const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
+      const expectedUrl = 'https://almonit.eth.link/'
+      const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
+      dnslinkResolver.clearCache()
+    })
   })
 
   describe('with recoverFailedHttpRequests=false', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = false
+      state.dnslinkPolicy = false
+    })
+    it('should do nothing on unreachable third party public gateway', async function () {
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
+    it('should do nothing on unreachable HTTP server with DNSLink', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
+      const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+      dnslinkResolver.clearCache()
     })
     it('should do nothing on failed non-default public gateway IPFS request', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('almonit.eth', false)
+      dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
+      const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
+      const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+      dnslinkResolver.clearCache()
     })
   })