test: add preload tests for ipfs.files.add and make fixes

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

.aegir.js

@@ -1,8 +1,11 @@
 'use strict'
 
-const createServer = require('ipfsd-ctl').createServer
+const IPFSFactory = require('ipfsd-ctl')
+const parallel = require('async/parallel')
+const MockPreloadNode = require('./test/utils/mock-preload-node')
 
-const server = createServer()
+const ipfsdServer = IPFSFactory.createServer()
+const preloadNode = MockPreloadNode.createNode()
 
 module.exports = {
   webpack: {
@@ -21,9 +24,29 @@ module.exports = {
     singleRun: true
   },
   hooks: {
+    node: {
+      pre: (cb) => preloadNode.start(cb),
+      post: (cb) => preloadNode.stop(cb)
+    },
     browser: {
-      pre: server.start.bind(server),
-      post: server.stop.bind(server)
+      pre: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.start()
+            cb()
+          },
+          (cb) => preloadNode.start(cb)
+        ], cb)
+      },
+      post: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.stop()
+            cb()
+          },
+          (cb) => preloadNode.stop(cb)
+        ], cb)
+      }
     }
   }
 }

README.md

@@ -231,11 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s
         - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`)
         - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`)
 
-- `preload` (object): Configure external nodes that will preload any content added to this node
+- `preload` (object): Configure external nodes that will preload content added to this node
     - `enabled` (boolean): Enable content preloading (Default: `false`)
-    - `addresses` (array): Bootstrap and gateway addresses for the preload nodes
-        - `bootstrap` (string): Multiaddr swarm addresses of a node that should preload content
-        - `gateway` (string): Multiaddr gateway addresses of a node that should preload content
+    - `gateways` (array): Multiaddr gateway addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap`
 - `EXPERIMENTAL` (object): Enable and configure experimental features.
     - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`)
     - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`)

src/core/components/files.js

@@ -89,6 +89,22 @@ function normalizeContent (opts, content) {
   })
 }
 
+function preloadFile (self, opts, file, cb) {
+  const isRootFile = opts.wrapWithDirectory
+    ? file.path === ''
+    : !file.path.includes('/')
+
+  const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false
+
+  if (!shouldPreload) return cb(null, file)
+
+  self._preload(new CID(file.hash), (err) => {
+    // Preload error is not fatal
+    if (err) console.error(err)
+    cb(null, file)
+  })
+}
+
 function pinFile (self, opts, file, cb) {
   // Pin a file if it is the root dir of a recursive add or the single file
   // of a direct add.
@@ -158,6 +174,7 @@ module.exports = function files (self) {
       pull.flatten(),
       importer(self._ipld, opts),
       pull.asyncMap(prepareFile.bind(null, self, opts)),
+      pull.asyncMap(preloadFile.bind(null, self, opts)),
       pull.asyncMap(pinFile.bind(null, self, opts))
     )
   }

src/core/components/libp2p.js

@@ -58,19 +58,6 @@ module.exports = function libp2p (self) {
           libp2pDefaults
         )
 
-        // Add the addresses for the preload nodes to the bootstrap addresses
-        if (get(self._options, 'preload.enabled')) {
-          let bootstrapList = libp2pOptions.config.peerDiscovery.bootstrap.list
-
-          const preloadBootstrap = get(self._options, 'preload.addresses', [])
-            .map(address => address.bootstrap)
-            .filter(Boolean) // A preload node doesn't _have_ to be added to the boostrap
-            .filter(address => !bootstrapList.includes(address)) // De-dupe
-
-          bootstrapList = bootstrapList.concat(preloadBootstrap)
-          libp2pOptions.config.peerDiscovery.bootstrap.list = bootstrapList
-        }
-
         self._libp2pNode = new Node(libp2pOptions)
 
         self._libp2pNode.on('peer:discovery', (peerInfo) => {

src/core/config.js

@@ -10,18 +10,12 @@ const schema = Joi.object().keys({
   repoOwner: Joi.boolean().default(true),
   preload: Joi.object().keys({
     enabled: Joi.boolean().default(false),
-    addresses: Joi.array()
-      .items(Joi.object().keys({
-        bootstrap: Joi.multiaddr().options({ convert: false }),
-        gateway: Joi.multiaddr().options({ convert: false })
-      }))
-      .default([{
-        bootstrap: '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-        gateway: '/dns4/wss0.bootstrap.libp2p.io/tcp/443'
-      }, {
-        bootstrap: '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-        gateway: '/dns4/wss1.bootstrap.libp2p.io/tcp/443'
-      }])
+    gateways: Joi.array()
+      .items(Joi.multiaddr().options({ convert: false }))
+      .default([
+        '/dns4/wss0.bootstrap.libp2p.io/https',
+        '/dns4/wss1.bootstrap.libp2p.io/https'
+      ])
   }).allow(null),
   init: Joi.alternatives().try(
     Joi.boolean(),

src/core/index.js

@@ -22,6 +22,7 @@ const boot = require('./boot')
 const components = require('./components')
 // replaced by repo-browser when running in the browser
 const defaultRepo = require('./runtime/repo-nodejs')
+const preload = require('./preload')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -78,6 +79,7 @@ class IPFS extends EventEmitter {
     this._blockService = new BlockService(this._repo)
     this._ipld = new Ipld(this._blockService)
     this._pubsub = undefined
+    this._preload = preload(this._options.preload)
 
     // IPFS Core exposed components
     //   - for booting up a node

src/core/preload.js

@@ -1,40 +1,51 @@
-const get = require('lodash/get')
+'use strict'
+
 const setImmediate = require('async/setImmediate')
 const each = require('async/each')
 const toUri = require('multiaddr-to-uri')
+const debug = require('debug')
 const preload = require('./runtime/preload-nodejs')
 
+const log = debug('jsipfs:preload')
+log.error = debug('jsipfs:preload:error')
+
 // Tools like IPFS Companion redirect requests to IPFS gateways to your local
 // gateway. This is a hint to those tools that they shouldn't redirect these
 // requests as they will effectively disable the preloading.
 const redirectOptOutHint = 'x-ipfs-preload'
 
-module.exports = self => {
-  const enabled = get(self._options, 'preload.enabled')
-  const gateways = get(self._options, 'preload.addresses', [])
-    .map(address => address.gateway)
-    .filter(Boolean)
+module.exports = (options) => {
+  options = options || {}
+  options.enabled = !!options.enabled
+  options.gateways = options.gateways || []
 
-  if (!enabled || !gateways.length) {
+  if (!options.enabled || !options.gateways.length) {
     return (_, callback) => {
-      if (!callback) return
-      setImmediate(() => callback())
+      if (callback) {
+        setImmediate(() => callback())
+      }
     }
   }
 
+  const noop = (err) => {
+    if (err) log.error(err)
+  }
+
   return (cid, callback) => {
-    each(gateways, (gatewayAddr, cb) => {
+    callback = callback || noop
+
+    each(options.gateways, (gatewayAddr, cb) => {
       let gatewayUri
 
       try {
         gatewayUri = toUri(gatewayAddr)
+        gatewayUri = gatewayUri.startsWith('http') ? gatewayUri : `http://${gatewayUri}`
       } catch (err) {
         return cb(err)
       }
 
-      const preloadUrl = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}`
-
-      preload(preloadUrl, cb)
+      const url = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}`
+      preload(url, cb)
     }, callback)
   }
 }

src/core/runtime/preload-browser.js

@@ -1,3 +1,6 @@
+/* eslint-env browser */
+'use strict'
+
 const debug = require('debug')
 
 const log = debug('jsipfs:preload')
@@ -6,7 +9,7 @@ log.error = debug('jsipfs:preload:error')
 module.exports = function preload (url, callback) {
   log(url)
 
-  const req = new window.XMLHttpRequest()
+  const req = new self.XMLHttpRequest()
 
   req.open('HEAD', url)
 
@@ -15,7 +18,7 @@ module.exports = function preload (url, callback) {
       return
     }
 
-    if (this.status !== 200) {
+    if (this.status < 200 || this.status >= 300) {
       log.error('failed to preload', url, this.status, this.statusText)
       return callback(new Error(`failed to preload ${url}`))
     }

src/core/runtime/preload-nodejs.js

@@ -1,14 +1,21 @@
+'use strict'
+
 const http = require('http')
-const URL = require('url')
+const { URL } = require('url')
 const debug = require('debug')
+const setImmediate = require('async/setImmediate')
 
 const log = debug('jsipfs:preload')
 log.error = debug('jsipfs:preload:error')
 
 module.exports = function preload (url, callback) {
   log(url)
 
-  url = new URL(url)
+  try {
+    url = new URL(url)
+  } catch (err) {
+    return setImmediate(() => callback(err))
+  }
 
   const req = http.request({
     protocol: url.protocol,
@@ -17,10 +24,13 @@ module.exports = function preload (url, callback) {
     path: url.pathname,
     method: 'HEAD'
   }, (res) => {
-    if (res.statusCode !== 200) {
+    res.resume()
+
+    if (res.statusCode < 200 || res.statusCode >= 300) {
       log.error('failed to preload', url, res.statusCode, res.statusMessage)
       return callback(new Error(`failed to preload ${url}`))
     }
+
     callback()
   })