feat: preload on content fetch requests (#1475)

When JS IPFS requests content not stored locally it needs to ask the peers it knows about to provide it if they have it. The peers are not relays so if they don't have it, they won't find it and provide it to JS IPFS. However, if we issue a preload request prior to these requests we prompt the preload nodes to fetch the content using their DHT and they can then provide it to JS IPFS.

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

* chore: update dependencies (#1473)

* chore: update libp2p and is-ipfs dependencies

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

* chore: update all the deps

* fix: rm non used boostrappers from tests

* test: increase mfs preload timeout

* test: pin-set timeout increase

* test: add preload tests for content retrieval

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

* fix: GET request not HEAD

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

package.json

@@ -58,7 +58,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^15.0.0",
+    "aegir": "^15.1.0",
     "buffer-loader": "~0.0.1",
     "chai": "^4.1.2",
     "delay": "^3.0.0",
@@ -70,8 +70,8 @@
     "expose-loader": "~0.7.5",
     "form-data": "^2.3.2",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.72.0",
-    "ipfsd-ctl": "~0.37.5",
+    "interface-ipfs-core": "~0.72.1",
+    "ipfsd-ctl": "~0.38.0",
     "mocha": "^5.2.0",
     "ncp": "^2.0.0",
     "nexpect": "~0.5.0",
@@ -93,7 +93,7 @@
     "byteman": "^1.3.5",
     "cids": "~0.5.3",
     "debug": "^3.1.0",
-    "file-type": "^8.0.0",
+    "file-type": "^8.1.0",
     "filesize": "^3.6.1",
     "fnv1a": "^1.0.1",
     "fsm-event": "^2.1.0",
@@ -104,30 +104,30 @@
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.4.2",
-    "ipfs-api": "^22.2.1",
-    "ipfs-bitswap": "~0.20.2",
+    "ipfs-api": "^22.2.4",
+    "ipfs-bitswap": "~0.20.3",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
-    "ipfs-mfs": "~0.2.2",
+    "ipfs-mfs": "~0.2.3",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
     "ipfs-unixfs-engine": "~0.31.3",
     "ipld": "~0.17.3",
     "ipld-dag-cbor": "~0.12.1",
-    "ipld-dag-pb": "~0.14.5",
-    "is-ipfs": "~0.3.2",
+    "ipld-dag-pb": "~0.14.6",
+    "is-ipfs": "~0.4.2",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^2.0.0",
-    "libp2p": "~0.22.0",
+    "libp2p": "~0.23.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-circuit": "~0.2.0",
     "libp2p-floodsub": "~0.15.0",
-    "libp2p-kad-dht": "~0.10.0",
+    "libp2p-kad-dht": "~0.10.1",
     "libp2p-keychain": "~0.3.1",
     "libp2p-mdns": "~0.12.0",
     "libp2p-mplex": "~0.8.0",
@@ -138,7 +138,7 @@
     "libp2p-websockets": "~0.12.0",
     "lodash": "^4.17.10",
     "mafmt": "^6.0.0",
-    "mime-types": "^2.1.18",
+    "mime-types": "^2.1.19",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
     "multiaddr-to-uri": "^4.0.0",

src/core/components/block.js

@@ -9,8 +9,20 @@ const promisify = require('promisify-es6')
 
 module.exports = function block (self) {
   return {
-    get: promisify((cid, callback) => {
+    get: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
       cid = cleanCid(cid)
+
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, callback)
     }),
     put: promisify((block, options, callback) => {
@@ -65,9 +77,18 @@ module.exports = function block (self) {
       cid = cleanCid(cid)
       self._blockService.delete(cid, callback)
     }),
-    stat: promisify((cid, callback) => {
+    stat: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
       cid = cleanCid(cid)
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, (err, block) => {
         if (err) {
           return callback(err)

src/core/components/dag.js

@@ -66,6 +66,10 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, path, options, callback)
     }),
 
@@ -100,17 +104,28 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       pull(
         self._ipld.treeStream(cid, path, options),
         pull.collect(callback)
       )
     }),
 
     // TODO - use IPLD selectors once they are implemented
-    _getRecursive: promisify((multihash, callback) => {
+    _getRecursive: promisify((multihash, options, callback) => {
       // gets flat array of all DAGNodes in tree given by multihash
 
-      self.dag.get(new CID(multihash), (err, res) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
+      self.dag.get(new CID(multihash), '', options, (err, res) => {
         if (err) { return callback(err) }
 
         mapAsync(res.value.links, (link, cb) => {

src/core/components/files.js

@@ -182,11 +182,17 @@ module.exports = function files (self) {
       throw new Error('You must supply an ipfsPath')
     }
 
+    options = options || {}
+
     ipfsPath = normalizePath(ipfsPath)
     const pathComponents = ipfsPath.split('/')
     const restPath = normalizePath(pathComponents.slice(1).join('/'))
     const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath)
 
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
+
     const d = deferred.source()
 
     pull(
@@ -213,16 +219,21 @@ module.exports = function files (self) {
   }
 
   function _lsPullStreamImmutable (ipfsPath, options) {
+    options = options || {}
+
     const path = normalizePath(ipfsPath)
-    const recursive = options && options.recursive
-    const pathDepth = path.split('/').length
+    const recursive = options.recursive
+    const pathComponents = path.split('/')
+    const pathDepth = pathComponents.length
     const maxDepth = recursive ? global.Infinity : pathDepth
-    const opts = Object.assign({}, {
-      maxDepth: maxDepth
-    }, options)
+    options.maxDepth = options.maxDepth || maxDepth
+
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
 
     return pull(
-      exporter(ipfsPath, self._ipld, opts),
+      exporter(ipfsPath, self._ipld, options),
       pull.filter(node =>
         recursive ? node.depth >= pathDepth : node.depth === pathDepth
       ),
@@ -334,8 +345,11 @@ module.exports = function files (self) {
         options = {}
       }
 
-      if (typeof callback !== 'function') {
-        throw new Error('Please supply a callback to ipfs.files.get')
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
       }
 
       pull(
@@ -359,6 +373,13 @@ module.exports = function files (self) {
     }),
 
     getReadableStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(
         pull(
           exporter(ipfsPath, self._ipld, options),
@@ -375,6 +396,13 @@ module.exports = function files (self) {
     },
 
     getPullStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return exporter(ipfsPath, self._ipld, options)
     },
 
@@ -384,6 +412,13 @@ module.exports = function files (self) {
         options = {}
       }
 
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       pull(
         _lsPullStreamImmutable(ipfsPath, options),
         pull.collect((err, values) => {
@@ -397,6 +432,13 @@ module.exports = function files (self) {
     }),
 
     lsReadableStreamImmutable: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
     },
 

src/core/components/mfs.js

@@ -7,7 +7,23 @@ module.exports = self => {
   const mfsSelf = Object.assign({}, self)
 
   // A patched dag API to ensure preload doesn't happen for MFS operations
+  // (MFS is preloaded periodically)
   mfsSelf.dag = Object.assign({}, self.dag, {
+    get: promisify((cid, path, opts, cb) => {
+      if (typeof path === 'function') {
+        cb = path
+        path = undefined
+      }
+
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      opts = Object.assign({}, opts, { preload: false })
+
+      return self.dag.get(cid, path, opts, cb)
+    }),
     put: promisify((node, opts, cb) => {
       if (typeof opts === 'function') {
         cb = opts

src/core/components/object.js

@@ -1,14 +1,14 @@
 'use strict'
 
 const waterfall = require('async/waterfall')
+const setImmediate = require('async/setImmediate')
 const promisify = require('promisify-es6')
 const dagPB = require('ipld-dag-pb')
 const DAGNode = dagPB.DAGNode
 const DAGLink = dagPB.DAGLink
 const CID = require('cids')
 const mh = require('multihashes')
 const Unixfs = require('ipfs-unixfs')
-const assert = require('assert')
 
 function normalizeMultihash (multihash, enc) {
   if (typeof multihash === 'string') {
@@ -116,7 +116,9 @@ module.exports = function object (self) {
       let data
 
       if (template) {
-        assert(template === 'unixfs-dir', 'unkown template')
+        if (template !== 'unixfs-dir') {
+          return setImmediate(() => callback(new Error('unknown template')))
+        }
         data = (new Unixfs('directory')).marshal()
       } else {
         data = Buffer.alloc(0)
@@ -221,6 +223,10 @@ module.exports = function object (self) {
         cid = cid.toV1()
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, (err, result) => {
         if (err) {
           return callback(err)

src/core/components/pin-set.js

@@ -74,7 +74,7 @@ exports = module.exports = function (dag) {
 
           seen[bs58Link] = true
 
-          dag.get(multihash, (err, res) => {
+          dag.get(multihash, '', { preload: false }, (err, res) => {
             if (err) { return someCb(err) }
             searchChildren(res.value, someCb)
           })
@@ -184,7 +184,7 @@ exports = module.exports = function (dag) {
         return callback(new Error('No link found with name ' + name))
       }
 
-      dag.get(link.multihash, (err, res) => {
+      dag.get(link.multihash, '', { preload: false }, (err, res) => {
         if (err) { return callback(err) }
         const keys = []
         const step = link => keys.push(link.multihash)
@@ -211,7 +211,7 @@ exports = module.exports = function (dag) {
 
           if (!emptyKey.equals(linkHash)) {
             // walk the links of this fanout bin
-            return dag.get(linkHash, (err, res) => {
+            return dag.get(linkHash, '', { preload: false }, (err, res) => {
               if (err) { return eachCb(err) }
               pinSet.walkItems(res.value, step, eachCb)
             })

src/core/components/pin.js

@@ -360,7 +360,7 @@ module.exports = (self) => {
         (_, cb) => repo.datastore.has(pinDataStoreKey, cb),
         (has, cb) => has ? cb() : cb(new Error('No pins to load')),
         (cb) => repo.datastore.get(pinDataStoreKey, cb),
-        (mh, cb) => dag.get(new CID(mh), cb)
+        (mh, cb) => dag.get(new CID(mh), '', { preload: false }, cb)
       ], (err, pinRoot) => {
         if (err) {
           if (err.message === 'No pins to load') {

src/core/runtime/preload-browser.js

@@ -11,7 +11,7 @@ module.exports = function preload (url, callback) {
 
   const req = new self.XMLHttpRequest()
 
-  req.open('HEAD', url)
+  req.open('GET', url)
 
   req.onreadystatechange = function () {
     if (this.readyState !== this.DONE) {