From 0a08192c54c5590b47aa09340ee47e6067e0c8a2 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Fri, 17 May 2019 08:46:25 +0200 Subject: [PATCH] feat: add support for File DOM API to files-regular (#2013) * feat: add support for File DOM API to files-regular * chore: use correct interface tests * chore: use correct interface tests * chore: fix package declaration cause npm is dumb * chore: fix deps * chore: fix deps * chore: fix deps * fix: use correct input * chore: add correct http-client * fix: use ipfs-utils * chore: fix semver lint --- package.json | 8 +++-- .../files-regular/add-pull-stream.js | 32 ++++++++++++------- src/core/components/files-regular/add.js | 23 +++---------- src/core/components/files-regular/utils.js | 16 +++++++--- 4 files changed, 43 insertions(+), 36 deletions(-) diff --git a/package.json b/package.json index 28571176db..da8a8d8006 100644 --- a/package.json +++ b/package.json @@ -72,12 +72,12 @@ "execa": "^1.0.0", "form-data": "^2.3.3", "hat": "0.0.3", - "interface-ipfs-core": "~0.101.1", "ipfsd-ctl": "~0.42.0", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", "qs": "^6.5.2", "rimraf": "^2.6.2", + "interface-ipfs-core": "~0.102.0", "sinon": "^7.3.1", "stream-to-promise": "^2.2.0" }, @@ -109,11 +109,11 @@ "glob": "^7.1.3", "hapi-pino": "^6.0.0", "human-to-milliseconds": "^1.0.0", - "interface-datastore": "~0.6.0", + "interface-datastore": "~0.7.0", "ipfs-bitswap": "~0.23.0", "ipfs-block": "~0.8.0", "ipfs-block-service": "~0.15.1", - "ipfs-http-client": "^31.0.2", + "ipfs-http-client": "^31.1.0", "ipfs-http-response": "~0.2.1", "ipfs-mfs": "~0.10.2", "ipfs-multipart": "~0.1.0", @@ -134,8 +134,10 @@ "is-pull-stream": "~0.0.0", "is-stream": "^2.0.0", "iso-url": "~0.4.6", + "ipfs-utils": "~0.0.3", "just-flatten-it": "^2.1.0", "just-safe-set": "^2.1.0", + "kind-of": "^6.0.2", "libp2p": "~0.25.0", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.16.0", diff --git a/src/core/components/files-regular/add-pull-stream.js b/src/core/components/files-regular/add-pull-stream.js index d69d2c8258..9937b08852 100644 --- a/src/core/components/files-regular/add-pull-stream.js +++ b/src/core/components/files-regular/add-pull-stream.js @@ -1,13 +1,20 @@ 'use strict' const importer = require('ipfs-unixfs-importer') -const pull = require('pull-stream') +const kindOf = require('kind-of') +const CID = require('cids') +const pullValues = require('pull-stream/sources/values') +const pullMap = require('pull-stream/throughs/map') +const pullAsyncMap = require('pull-stream/throughs/async-map') +const pullFlatten = require('pull-stream/throughs/flatten') +const pull = require('pull-stream/pull') const toPull = require('stream-to-pull-stream') const waterfall = require('async/waterfall') const isStream = require('is-stream') -const isSource = require('is-pull-stream').isSource -const CID = require('cids') +const { isSource } = require('is-pull-stream') const { parseChunkerString } = require('./utils') +const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader') +const { supportsFileReader } = require('ipfs-utils/src/supports') const WRAPPER = 'wrapper/' @@ -52,9 +59,12 @@ function normalizeContent (content, opts) { } return content.map((data) => { + if (supportsFileReader && kindOf(data) === 'file') { + data = { path: '', content: toPull.source(streamFromFileReader(data)) } + } // Buffer input if (Buffer.isBuffer(data)) { - data = { path: '', content: pull.values([data]) } + data = { path: '', content: pullValues([data]) } } // Readable stream input @@ -68,7 +78,7 @@ function normalizeContent (content, opts) { if (data && data.content && typeof data.content !== 'function') { if (Buffer.isBuffer(data.content)) { - data.content = pull.values([data.content]) + data.content = pullValues([data.content]) } if (isStream.readable(data.content)) { @@ -124,7 +134,7 @@ module.exports = function (self) { try { chunkerOptions = parseChunkerString(options.chunker) } catch (err) { - return pull.map(() => { throw err }) + return pullMap(() => { throw err }) } const opts = Object.assign({}, { shardSplitThreshold: self._options.EXPERIMENTAL.sharding @@ -147,12 +157,12 @@ module.exports = function (self) { opts.progress = progress return pull( - pull.map(content => normalizeContent(content, opts)), - pull.flatten(), + pullMap(content => normalizeContent(content, opts)), + pullFlatten(), importer(self._ipld, opts), - pull.asyncMap((file, cb) => prepareFile(file, self, opts, cb)), - pull.map(file => preloadFile(file, self, opts)), - pull.asyncMap((file, cb) => pinFile(file, self, opts, cb)) + pullAsyncMap((file, cb) => prepareFile(file, self, opts, cb)), + pullMap(file => preloadFile(file, self, opts)), + pullAsyncMap((file, cb) => pinFile(file, self, opts, cb)) ) } } diff --git a/src/core/components/files-regular/add.js b/src/core/components/files-regular/add.js index 174cf8d51a..d254159b7f 100644 --- a/src/core/components/files-regular/add.js +++ b/src/core/components/files-regular/add.js @@ -3,8 +3,8 @@ const promisify = require('promisify-es6') const pull = require('pull-stream') const sort = require('pull-sort') -const isStream = require('is-stream') const isSource = require('is-pull-stream').isSource +const validateAddInput = require('ipfs-utils/src/files/add-input-validation') module.exports = function (self) { const add = promisify((data, options, callback) => { @@ -15,23 +15,10 @@ module.exports = function (self) { options = options || {} - // Buffer, pull stream or Node.js stream - const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj) - // An object like { content?, path? }, where content isBufferOrStream and path isString - const isContentObject = obj => { - if (typeof obj !== 'object') return false - // path is optional if content is present - if (obj.content) return isBufferOrStream(obj.content) - // path must be a non-empty string if no content - return Boolean(obj.path) && typeof obj.path === 'string' - } - // An input atom: a buffer, stream or content object - const isInput = obj => isBufferOrStream(obj) || isContentObject(obj) - // All is ok if data isInput or data is an array of isInput - const ok = isInput(data) || (Array.isArray(data) && data.every(isInput)) - - if (!ok) { - return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects')) + try { + validateAddInput(data) + } catch (err) { + return callback(err) } pull( diff --git a/src/core/components/files-regular/utils.js b/src/core/components/files-regular/utils.js index 8b1241ae76..abda5f6d85 100644 --- a/src/core/components/files-regular/utils.js +++ b/src/core/components/files-regular/utils.js @@ -1,8 +1,9 @@ 'use strict' const CID = require('cids') +const { Buffer } = require('buffer') -exports.normalizePath = (path) => { +const normalizePath = (path) => { if (Buffer.isBuffer(path)) { return new CID(path).toString() } @@ -30,7 +31,7 @@ exports.normalizePath = (path) => { * * @return {Object} Chunker options for DAGBuilder */ -exports.parseChunkerString = (chunker) => { +const parseChunkerString = (chunker) => { if (!chunker) { return { chunker: 'fixed' @@ -67,7 +68,7 @@ exports.parseChunkerString = (chunker) => { * * @return {Object} rabin chunker options */ -function parseRabinString (chunker) { +const parseRabinString = (chunker) => { const options = {} const parts = chunker.split('-') switch (parts.length) { @@ -89,7 +90,7 @@ function parseRabinString (chunker) { return options } -function parseChunkSize (str, name) { +const parseChunkSize = (str, name) => { let size = parseInt(str) if (isNaN(size)) { throw new Error(`Chunker parameter ${name} must be an integer`) @@ -97,3 +98,10 @@ function parseChunkSize (str, name) { return size } + +module.exports = { + normalizePath, + parseChunkSize, + parseRabinString, + parseChunkerString +}