From 25a8c4f321ebab1eb87a88615167aee3ce11c2da Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 25 Jul 2018 16:30:57 +0100 Subject: [PATCH] test: add preload tests for ipfs.files.add and make fixes License: MIT Signed-off-by: Alan Shaw --- .aegir.js | 31 ++++++- README.md | 6 +- src/core/components/files.js | 17 ++++ src/core/components/libp2p.js | 13 --- src/core/config.js | 18 ++--- src/core/index.js | 2 + src/core/preload.js | 37 ++++++--- src/core/runtime/preload-browser.js | 7 +- src/core/runtime/preload-nodejs.js | 16 +++- test/core/preload.spec.js | 120 ++++++++++++++++++++++++++++ test/utils/mock-preload-node.js | 120 ++++++++++++++++++++++++++++ 11 files changed, 336 insertions(+), 51 deletions(-) create mode 100644 test/core/preload.spec.js create mode 100644 test/utils/mock-preload-node.js diff --git a/.aegir.js b/.aegir.js index 979ffde3df..5d87aa66d7 100644 --- a/.aegir.js +++ b/.aegir.js @@ -1,8 +1,11 @@ 'use strict' -const createServer = require('ipfsd-ctl').createServer +const IPFSFactory = require('ipfsd-ctl') +const parallel = require('async/parallel') +const MockPreloadNode = require('./test/utils/mock-preload-node') -const server = createServer() +const ipfsdServer = IPFSFactory.createServer() +const preloadNode = MockPreloadNode.createNode() module.exports = { webpack: { @@ -21,9 +24,29 @@ module.exports = { singleRun: true }, hooks: { + node: { + pre: (cb) => preloadNode.start(cb), + post: (cb) => preloadNode.stop(cb) + }, browser: { - pre: server.start.bind(server), - post: server.stop.bind(server) + pre: (cb) => { + parallel([ + (cb) => { + ipfsdServer.start() + cb() + }, + (cb) => preloadNode.start(cb) + ], cb) + }, + post: (cb) => { + parallel([ + (cb) => { + ipfsdServer.stop() + cb() + }, + (cb) => preloadNode.stop(cb) + ], cb) + } } } } diff --git a/README.md b/README.md index 0f45d25fcf..e312940788 100644 --- a/README.md +++ b/README.md @@ -231,11 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) -- `preload` (object): Configure external nodes that will preload any content added to this node +- `preload` (object): Configure external nodes that will preload content added to this node - `enabled` (boolean): Enable content preloading (Default: `false`) - - `addresses` (array): Bootstrap and gateway addresses for the preload nodes - - `bootstrap` (string): Multiaddr swarm addresses of a node that should preload content - - `gateway` (string): Multiaddr gateway addresses of a node that should preload content + - `gateways` (array): Multiaddr gateway addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap` - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) diff --git a/src/core/components/files.js b/src/core/components/files.js index f69868bd77..632da9e154 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -89,6 +89,22 @@ function normalizeContent (opts, content) { }) } +function preloadFile (self, opts, file, cb) { + const isRootFile = opts.wrapWithDirectory + ? file.path === '' + : !file.path.includes('/') + + const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false + + if (!shouldPreload) return cb(null, file) + + self._preload(new CID(file.hash), (err) => { + // Preload error is not fatal + if (err) console.error(err) + cb(null, file) + }) +} + function pinFile (self, opts, file, cb) { // Pin a file if it is the root dir of a recursive add or the single file // of a direct add. @@ -158,6 +174,7 @@ module.exports = function files (self) { pull.flatten(), importer(self._ipld, opts), pull.asyncMap(prepareFile.bind(null, self, opts)), + pull.asyncMap(preloadFile.bind(null, self, opts)), pull.asyncMap(pinFile.bind(null, self, opts)) ) } diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 4ee772d19d..1ad0d1ecac 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -58,19 +58,6 @@ module.exports = function libp2p (self) { libp2pDefaults ) - // Add the addresses for the preload nodes to the bootstrap addresses - if (get(self._options, 'preload.enabled')) { - let bootstrapList = libp2pOptions.config.peerDiscovery.bootstrap.list - - const preloadBootstrap = get(self._options, 'preload.addresses', []) - .map(address => address.bootstrap) - .filter(Boolean) // A preload node doesn't _have_ to be added to the boostrap - .filter(address => !bootstrapList.includes(address)) // De-dupe - - bootstrapList = bootstrapList.concat(preloadBootstrap) - libp2pOptions.config.peerDiscovery.bootstrap.list = bootstrapList - } - self._libp2pNode = new Node(libp2pOptions) self._libp2pNode.on('peer:discovery', (peerInfo) => { diff --git a/src/core/config.js b/src/core/config.js index 0c8342efad..7f2b82f925 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -10,18 +10,12 @@ const schema = Joi.object().keys({ repoOwner: Joi.boolean().default(true), preload: Joi.object().keys({ enabled: Joi.boolean().default(false), - addresses: Joi.array() - .items(Joi.object().keys({ - bootstrap: Joi.multiaddr().options({ convert: false }), - gateway: Joi.multiaddr().options({ convert: false }) - })) - .default([{ - bootstrap: '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - gateway: '/dns4/wss0.bootstrap.libp2p.io/tcp/443' - }, { - bootstrap: '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - gateway: '/dns4/wss1.bootstrap.libp2p.io/tcp/443' - }]) + gateways: Joi.array() + .items(Joi.multiaddr().options({ convert: false })) + .default([ + '/dns4/wss0.bootstrap.libp2p.io/https', + '/dns4/wss1.bootstrap.libp2p.io/https' + ]) }).allow(null), init: Joi.alternatives().try( Joi.boolean(), diff --git a/src/core/index.js b/src/core/index.js index 36f7c3a118..ad9c9af6ba 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -22,6 +22,7 @@ const boot = require('./boot') const components = require('./components') // replaced by repo-browser when running in the browser const defaultRepo = require('./runtime/repo-nodejs') +const preload = require('./preload') class IPFS extends EventEmitter { constructor (options) { @@ -78,6 +79,7 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld(this._blockService) this._pubsub = undefined + this._preload = preload(this._options.preload) // IPFS Core exposed components // - for booting up a node diff --git a/src/core/preload.js b/src/core/preload.js index 4293c856a5..dcefc52150 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -1,40 +1,51 @@ -const get = require('lodash/get') +'use strict' + const setImmediate = require('async/setImmediate') const each = require('async/each') const toUri = require('multiaddr-to-uri') +const debug = require('debug') const preload = require('./runtime/preload-nodejs') +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + // Tools like IPFS Companion redirect requests to IPFS gateways to your local // gateway. This is a hint to those tools that they shouldn't redirect these // requests as they will effectively disable the preloading. const redirectOptOutHint = 'x-ipfs-preload' -module.exports = self => { - const enabled = get(self._options, 'preload.enabled') - const gateways = get(self._options, 'preload.addresses', []) - .map(address => address.gateway) - .filter(Boolean) +module.exports = (options) => { + options = options || {} + options.enabled = !!options.enabled + options.gateways = options.gateways || [] - if (!enabled || !gateways.length) { + if (!options.enabled || !options.gateways.length) { return (_, callback) => { - if (!callback) return - setImmediate(() => callback()) + if (callback) { + setImmediate(() => callback()) + } } } + const noop = (err) => { + if (err) log.error(err) + } + return (cid, callback) => { - each(gateways, (gatewayAddr, cb) => { + callback = callback || noop + + each(options.gateways, (gatewayAddr, cb) => { let gatewayUri try { gatewayUri = toUri(gatewayAddr) + gatewayUri = gatewayUri.startsWith('http') ? gatewayUri : `http://${gatewayUri}` } catch (err) { return cb(err) } - const preloadUrl = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` - - preload(preloadUrl, cb) + const url = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` + preload(url, cb) }, callback) } } diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js index dd765395c2..b80c8901a5 100644 --- a/src/core/runtime/preload-browser.js +++ b/src/core/runtime/preload-browser.js @@ -1,3 +1,6 @@ +/* eslint-env browser */ +'use strict' + const debug = require('debug') const log = debug('jsipfs:preload') @@ -6,7 +9,7 @@ log.error = debug('jsipfs:preload:error') module.exports = function preload (url, callback) { log(url) - const req = new window.XMLHttpRequest() + const req = new self.XMLHttpRequest() req.open('HEAD', url) @@ -15,7 +18,7 @@ module.exports = function preload (url, callback) { return } - if (this.status !== 200) { + if (this.status < 200 || this.status >= 300) { log.error('failed to preload', url, this.status, this.statusText) return callback(new Error(`failed to preload ${url}`)) } diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js index 8f09caca5e..7c20df4d3a 100644 --- a/src/core/runtime/preload-nodejs.js +++ b/src/core/runtime/preload-nodejs.js @@ -1,6 +1,9 @@ +'use strict' + const http = require('http') -const URL = require('url') +const { URL } = require('url') const debug = require('debug') +const setImmediate = require('async/setImmediate') const log = debug('jsipfs:preload') log.error = debug('jsipfs:preload:error') @@ -8,7 +11,11 @@ log.error = debug('jsipfs:preload:error') module.exports = function preload (url, callback) { log(url) - url = new URL(url) + try { + url = new URL(url) + } catch (err) { + return setImmediate(() => callback(err)) + } const req = http.request({ protocol: url.protocol, @@ -17,10 +24,13 @@ module.exports = function preload (url, callback) { path: url.pathname, method: 'HEAD' }, (res) => { - if (res.statusCode !== 200) { + res.resume() + + if (res.statusCode < 200 || res.statusCode >= 300) { log.error('failed to preload', url, res.statusCode, res.statusMessage) return callback(new Error(`failed to preload ${url}`)) } + callback() }) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js new file mode 100644 index 0000000000..1068dba05f --- /dev/null +++ b/test/core/preload.spec.js @@ -0,0 +1,120 @@ +/* eslint-env mocha */ +'use strict' + +const hat = require('hat') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const MockPreloadNode = require('../utils/mock-preload-node') +const IPFS = require('../../src') + +describe('preload', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + config: { + Addresses: { + Swarm: [] + } + }, + preload: { + enabled: true, + gateways: [MockPreloadNode.defaultAddr] + } + }) + + ipfs.on('ready', done) + }) + + afterEach((done) => MockPreloadNode.clearPreloadUrls(done)) + + after((done) => ipfs.stop(done)) + + it('should preload content added with ipfs.files.add', (done) => { + ipfs.files.add(Buffer.from(hat()), (err, res) => { + expect(err).to.not.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${res[0].hash}`) + done() + }) + }) + }) + + it('should preload multiple content added with ipfs.files.add', (done) => { + ipfs.files.add([{ + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(res.length) + res.forEach(file => { + const url = urls.find(url => url === `/ipfs/${file.hash}`) + expect(url).to.exist() + }) + done() + }) + }) + }) + + it('should preload root dir for multiple content added with ipfs.files.add', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + const rootDir = res.find(file => file.path === 'dir0') + expect(rootDir).to.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${rootDir.hash}`) + done() + }) + }) + }) + + it('should preload wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], { wrapWithDirectory: true }, (err, res) => { + expect(err).to.not.exist() + + const wrappingDir = res.find(file => file.path === '') + expect(wrappingDir).to.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${wrappingDir.hash}`) + done() + }) + }) + }) +}) diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js new file mode 100644 index 0000000000..2ca78c33a3 --- /dev/null +++ b/test/utils/mock-preload-node.js @@ -0,0 +1,120 @@ +/* eslint-env browser */ +'use strict' + +const http = require('http') +const toUri = require('multiaddr-to-uri') +const URL = require('url').URL || self.URL + +const defaultPort = 1138 +const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}/http` + +module.exports.defaultAddr = defaultAddr + +// Create a mock preload IPFS node with a gateway that'll respond 204 to a HEAD +// request. It also remembers the preload URLs it has been called with, and you +// can ask it for them and also clear them by issuing a GET/DELETE request. +module.exports.createNode = () => { + let urls = [] + + const server = http.createServer((req, res) => { + switch (req.method) { + case 'HEAD': + res.statusCode = 204 + urls = urls.concat(req.url) + break + case 'DELETE': + res.statusCode = 204 + urls = [] + break + case 'GET': + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify(urls)) + break + default: + res.statusCode = 500 + } + + res.end() + }) + + server.start = (opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + return server.listen(Object.assign({ port: defaultPort }, opts), cb) + } + + server.stop = (cb) => server.close(cb) + + return server +} + +function parseMultiaddr (addr) { + let url = toUri(addr) + url = url.startsWith('http://') ? url : `http://${url}` + return new URL(url) +} + +// Get the stored preload URLs for the server at `addr` +module.exports.getPreloadUrls = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.get({ protocol, hostname, port }, (res) => { + if (res.statusCode !== 200) { + res.resume() + return cb(new Error('failed to get preloaded URLs from mock preload node')) + } + + let data = '' + + res.on('error', cb) + res.on('data', chunk => { data += chunk }) + + res.on('end', () => { + let obj + try { + obj = JSON.parse(data) + } catch (err) { + return cb(err) + } + cb(null, obj) + }) + }) + + req.on('error', cb) +} + +// Clear the stored preload URLs for the server at `addr` +module.exports.clearPreloadUrls = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.request({ + method: 'DELETE', + protocol, + hostname, + port + }, (res) => { + res.resume() + + if (res.statusCode !== 204) { + return cb(new Error('failed to reset mock preload node')) + } + + cb() + }) + + req.on('error', cb) + req.end() +}