From 0af2ccdea09e1665b65103886d51f470870820b0 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 28 Dec 2015 01:42:35 +0100 Subject: [PATCH] Working and passing --- package.json | 14 ++-- src/get-files-stream.js | 152 ++++++++++++++++++++++------------------ src/load-commands.js | 5 +- src/request-api.js | 2 +- test/api/add.spec.js | 24 +++---- test/api/files.spec.js | 8 ++- test/setup.js | 2 +- 7 files changed, 109 insertions(+), 98 deletions(-) diff --git a/package.json b/package.json index f665a6153..bc0820ecc 100644 --- a/package.json +++ b/package.json @@ -4,15 +4,13 @@ "description": "A client library for the IPFS API", "main": "src/index.js", "dependencies": { - "merge-stream": "^1.0.0", + "detect-node": "^2.0.3", + "flatmap": "0.0.3", + "glob": "^6.0.2", "multiaddr": "^1.0.0", - "multipart-stream": "^2.0.0", + "multipart-stream": "github:dignifiedquire/multipart-stream#a1a4e9c4d41d1f78dcf32279affbbb0bfd930c28", "ndjson": "^1.4.3", "qs": "^6.0.0", - "require-dir": "^0.3.0", - "vinyl": "^1.1.0", - "vinyl-fs-browser": "^2.1.1-1", - "vinyl-multipart-stream": "^1.2.6", "wreck": "^7.0.0" }, "engines": { @@ -58,13 +56,11 @@ "pre-commit": "^1.0.6", "raw-loader": "^0.5.1", "rimraf": "^2.4.3", - "run-sequence": "^1.1.4", + "run-sequence": "^1.1.5", "semver": "^5.1.0", "stream-equal": "^0.1.7", "stream-http": "^2.0.2", "uglify-js": "^2.4.24", - "vinyl-buffer": "^1.0.0", - "vinyl-source-stream": "^1.1.0", "webpack-stream": "^3.1.0" }, "scripts": { diff --git a/src/get-files-stream.js b/src/get-files-stream.js index d48555316..9253d91ca 100644 --- a/src/get-files-stream.js +++ b/src/get-files-stream.js @@ -1,88 +1,104 @@ 'use strict' -const File = require('vinyl') -const vinylfs = require('vinyl-fs-browser') -const vmps = require('vinyl-multipart-stream') -const stream = require('stream') -const Merge = require('merge-stream') +const isNode = require('detect-node') +const Multipart = require('multipart-stream') +const flatmap = require('flatmap') -exports = module.exports = getFilesStream +function headers (file) { + const name = file.path || '' + const header = { + 'Content-Disposition': `file; filename="${name}"` + } -function getFilesStream (files, opts) { - if (!files) return null + if (file.dir) { + header['Content-Type'] = 'application/x-directory' + } else { + header['Content-Type'] = 'application/octet-stream' + } - // merge all inputs into one stream - const adder = new Merge() + return header +} - // single stream for pushing directly - const single = new stream.PassThrough({objectMode: true}) - adder.add(single) +function strip (name, base) { + const smallBase = base + .split('/') + .slice(0, -1) + .join('/') + '/' + return name.replace(smallBase, '') +} - for (let i = 0; i < files.length; i++) { - const file = files[i] +function loadPaths (opts, file) { + const path = require('path') + const fs = require('fs') + const glob = require('glob') - if (typeof (file) === 'string') { - const srcOpts = { - buffer: false, - stripBOM: false, - followSymlinks: opts.followSymlinks != null ? opts.followSymlinks : true - } + const followSymlinks = opts.followSymlinks != null ? opts.followSymlinks : true - // add the file or dir itself - adder.add(vinylfs.src(file, srcOpts)) + file = path.resolve(file) + const stats = fs.statSync(file) - // if recursive, glob the contents - if (opts.recursive) { - adder.add(vinylfs.src(file + '/**/*', srcOpts)) - } - } else { - // try to create a single vinyl file, and push it. - // throws if cannot use the file. - single.push(vinylFile(file)) - } + if (stats.isDirectory() && !opts.recursive) { + throw new Error('Can only add directories using --recursive') } - single.end() - return adder.pipe(vmps.flat()) -} + if (stats.isDirectory() && opts.recursive) { + const mg = new glob.sync.GlobSync(`${file}/**/*`, { + follow: followSymlinks + }) -// vinylFile tries to cast a file object to a vinyl file. -// it's agressive. If it _cannot_ be converted to a file, -// it returns null. -function vinylFile (file) { - if (file instanceof File) { - return file // it's a vinyl file. + return mg.found.map(name => { + if (mg.cache[name] === 'FILE') { + return { + path: strip(name, file), + dir: false, + content: fs.createReadStream(name) + } + } else { + return { + path: strip(name, file), + dir: true + } + } + }) } - // let's try to make a vinyl file? - const f = {cwd: '/', base: '/', path: ''} - if (file.contents && file.path) { - // set the cwd + base, if there. - f.path = file.path - f.cwd = file.cwd || f.cwd - f.base = file.base || f.base - f.contents = file.contents - } else { - // ok maybe we just have contents? - f.contents = file + return { + path: file, + content: fs.createReadStream(file) } - - // ensure the contents are safe to pass. - // throws if vinyl cannot use the contents - f.contents = vinylContentsSafe(f.contents) - return new File(f) } -function vinylContentsSafe (c) { - if (Buffer.isBuffer(c)) return c - if (typeof (c) === 'string') return c - if (c instanceof stream.Stream) return c - if (typeof (c.pipe) === 'function') { - // hey, looks like a stream. but vinyl won't detect it. - // pipe it to a PassThrough, and use that - const s = new stream.PassThrough() - return c.pipe(s) - } +function getFilesStream (files, opts) { + if (!files) return null + + const mp = new Multipart() + + flatmap(files, file => { + if (typeof file === 'string') { + if (!isNode) { + throw new Error('Can not add paths in node') + } + + return loadPaths(opts, file) + } + + if (file.path && (file.content || file.dir)) { + return file + } - throw new Error('vinyl will not accept: ' + c) + return { + path: '', + dir: false, + content: file + } + }).forEach(file => { + mp.addPart({ + headers: headers(file), + body: file.content + }) + }) + + return mp } + +exports = module.exports = getFilesStream diff --git a/src/load-commands.js b/src/load-commands.js index f6be472ac..7f997a057 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -1,10 +1,6 @@ 'use strict' -const isNode = !global.window - function requireCommands () { - if (isNode) return require('require-dir')('./api') - return { add: require('./api/add'), block: require('./api/block'), @@ -14,6 +10,7 @@ function requireCommands () { dht: require('./api/dht'), diag: require('./api/diag'), id: require('./api/id'), + files: require('./api/files'), log: require('./api/log'), ls: require('./api/ls'), mount: require('./api/mount'), diff --git a/src/request-api.js b/src/request-api.js index a619eed2d..68a07b481 100644 --- a/src/request-api.js +++ b/src/request-api.js @@ -5,7 +5,7 @@ const Qs = require('qs') const ndjson = require('ndjson') const getFilesStream = require('./get-files-stream') -const isNode = !global.window +const isNode = require('detect-node') // -- Internal diff --git a/test/api/add.spec.js b/test/api/add.spec.js index bfd61d67a..0defb23e4 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -1,10 +1,8 @@ 'use strict' -const path = require('path') -const File = require('vinyl') const Readable = require('stream').Readable -const isNode = !global.window +const isNode = require('detect-node') const testfilePath = __dirname + '/../testfile.txt' let testfile @@ -26,19 +24,17 @@ describe('.add', () => { return done() } - const file = new File({ - cwd: path.dirname(testfilePath), - base: path.dirname(testfilePath), - path: testfilePath, - contents: new Buffer(testfile) - }) + const file = { + path: 'testfile.txt', + content: new Buffer(testfile) + } - apiClients['a'].add(file, (err, res) => { + apiClients['a'].add([file], (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res expect(added).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(added).to.have.property('Name', path.basename(testfilePath)) + expect(added).to.have.property('Name', 'testfile.txt') done() }) }) @@ -47,6 +43,7 @@ describe('.add', () => { let buf = new Buffer(testfile) apiClients['a'].add(buf, (err, res) => { expect(err).to.not.exist + expect(res).to.have.length(1) expect(res[0]).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') done() @@ -60,6 +57,7 @@ describe('.add', () => { apiClients['a'].add(testfileBig, (err, res) => { expect(err).to.not.exist + expect(res).to.have.length(1) expect(res[0]).to.have.a.property('Hash', 'Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq') done() @@ -84,10 +82,9 @@ describe('.add', () => { apiClients['a'].add(__dirname + '/../test-folder', { recursive: true }, (err, res) => { if (isNode) { expect(err).to.not.exist - console.log('Jeromy ->', res) const added = res[res.length - 1] - expect(added).to.have.property('Hash', 'QmSzLpCVbWnEm3XoTWnv6DT6Ju5BsVoLhzvxKXZeQ2cmdg') + expect(added).to.have.property('Hash', 'QmTDH2RXGn8XyDAo9YyfbZAUXwL1FCr44YJCN9HBZmL9Gj') done() } else { expect(err.message).to.be.equal('Recursive uploads are not supported in the browser') @@ -100,6 +97,7 @@ describe('.add', () => { const stream = new Readable() stream.push('Hello world') stream.push(null) + apiClients['a'].add(stream, (err, res) => { expect(err).to.not.exist diff --git a/test/api/files.spec.js b/test/api/files.spec.js index e57791750..57689d472 100644 --- a/test/api/files.spec.js +++ b/test/api/files.spec.js @@ -69,14 +69,18 @@ describe('.files', function () { it('files.read', function (done) { this.timeout(20000) + if (!isNode) { + return done() + } + apiClients['a'].files.read('/test-folder/test-file', function (err, stream) { expect(err).to.not.exist let buf = '' stream - .on('error', function (err) { + .on('error', function (err) { expect(err).to.not.exist }) - .on('data', function (data) { + .on('data', function (data) { buf += data }) .on('end', function () { diff --git a/test/setup.js b/test/setup.js index 81ad12929..c15c85069 100644 --- a/test/setup.js +++ b/test/setup.js @@ -5,7 +5,7 @@ const apiAddrs = require('./tmp-disposable-nodes-addrs.json') global.expect = require('chai').expect global.apiClients = {} // a, b, c -global.isNode = !global.window +global.isNode = require('detect-node') function connectNodes (done) { const addrs = {}