From 9e84ef896e88ad0abcd40cf114690b662135c3b2 Mon Sep 17 00:00:00 2001 From: nginnever Date: Fri, 6 May 2016 09:25:37 -0700 Subject: [PATCH 1/6] files add, cat, get core + cli --- package.json | 21 ++--- src/cli/commands/files/add.js | 89 +++++++++++++++++-- src/cli/commands/files/cat.js | 38 ++++++++ src/cli/commands/files/get.js | 67 ++++++++++++++ src/core/ipfs/files.js | 56 ++++++++++-- src/core/ipfs/init.js | 42 ++++++++- test/cli-tests/test-commands.js | 2 +- test/core-tests/test-files.js | 64 +++++++++++++ test/core-tests/test-init-node.js | 14 +-- ...dc5e844ead5f91f12858b021eba45768b4c0e.data | 3 + 10 files changed, 359 insertions(+), 37 deletions(-) create mode 100644 src/cli/commands/files/cat.js create mode 100644 src/cli/commands/files/get.js create mode 100644 test/core-tests/test-files.js create mode 100644 test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data diff --git a/package.json b/package.json index 98bce8b23f..a1e79eeac7 100644 --- a/package.json +++ b/package.json @@ -44,39 +44,40 @@ "expose-loader": "^0.7.1", "form-data": "^1.0.0-rc3", "idb-plus-blob-store": "^1.1.2", - "lodash": "^4.11.1", - "mocha": "^2.3.4", + "lodash": "^4.11.2", + "mocha": "^2.4.5", "ncp": "^2.0.0", "nexpect": "^0.5.0", "pre-commit": "^1.1.2", - "rimraf": "^2.4.4", + "rimraf": "^2.5.2", "stream-to-promise": "^1.1.0", "transform-loader": "^0.2.3" }, "dependencies": { "babel-runtime": "^6.6.1", "bl": "^1.1.2", - "boom": "^3.1.1", + "boom": "^3.1.2", "bs58": "^3.0.0", "debug": "^2.2.0", "fs-blob-store": "^5.2.1", "glob": "^7.0.3", "hapi": "^13.3.0", - "ipfs-api": "^3.0.1", + "ipfs-api": "^3.0.2", "ipfs-block": "^0.3.0", "ipfs-block-service": "^0.3.0", - "ipfs-data-importing": "^0.3.3", "ipfs-merkle-dag": "^0.5.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", - "joi": "^8.0.2", - "libp2p-ipfs": "^0.3.3", + "ipfs-unixfs-engine": "^0.6.1", + "joi": "^8.0.5", + "libp2p-ipfs": "^0.3.5", "lodash.get": "^4.2.1", - "lodash.set": "^4.0.0", - "multiaddr": "^1.3.0", + "lodash.set": "^4.1.0", + "multiaddr": "^1.4.1", "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", + "readable-stream": "^1.1.13", "ronin": "^0.3.11", "temp": "^0.8.3" }, diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 5a0fd79138..27ec59ee8c 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -1,11 +1,45 @@ 'use strict' const Command = require('ronin').Command -const IPFS = require('../../../core') +const utils = require('../../utils') const debug = require('debug') const log = debug('cli:version') log.error = debug('cli:version:error') const bs58 = require('bs58') +const fs = require('fs') +const async = require('async') +const path = require('path') +const glob = require('glob') + +function checkPath (inPath, recursive) { + // This function is to check for the following possible inputs + // 1) "." add the cwd but throw error for no recursion flag + // 2) "." -r return the cwd + // 3) "/some/path" but throw error for no recursion + // 4) "/some/path" -r + // 5) No path, throw err + // 6) filename.type return the cwd + filename + + if (!inPath) { + throw new Error('Error: Argument \'path\' is required') + } + + var s = fs.statSync(inPath) + + if (s.isDirectory() && recursive === false) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + if (inPath === '.' && recursive === true) { + inPath = process.cwd() + return inPath + } else if (inPath === '.' && recursive === false) { + s = fs.statSync(process.cwd()) + if (s.isDirectory()) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + } + return inPath +} module.exports = Command.extend({ desc: 'Add a file to IPFS using the UnixFS data format', @@ -18,16 +52,53 @@ module.exports = Command.extend({ } }, - run: (recursive, path) => { - var node = new IPFS() - path = process.cwd() + '/' + path - node.files.add(path, { - recursive: recursive - }, (err, stats) => { + run: (recursive, inPath) => { + let rs + + inPath = checkPath(inPath, recursive) + + glob(path.join(inPath, '/**/*'), (err, res) => { if (err) { - return console.log(err) + throw err } - console.log('added', bs58.encode(stats.Hash).toString(), stats.Name) + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + const i = ipfs.files.add() + var filePair + i.on('data', (file) => { + console.log('added', bs58.encode(file.multihash).toString(), file.path) + }) + i.once('end', () => { + return + }) + if (res.length !== 0) { + const index = inPath.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + rs = fs.createReadStream(element) + filePair = {path: addPath, stream: rs} + i.write(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + i.end() + }) + } else { + rs = fs.createReadStream(inPath) + inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length) + filePair = {path: inPath, stream: rs} + i.write(filePair) + i.end() + } + }) }) } }) diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js new file mode 100644 index 0000000000..52dc3c1dbe --- /dev/null +++ b/src/cli/commands/files/cat.js @@ -0,0 +1,38 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + } + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.cat(path, (err, res) => { + if (err) { + throw (err) + } + if (res) { + res.on('file', (data) => { + data.stream.pipe(process.stdout) + }) + } + return + }) + }) + } +}) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js new file mode 100644 index 0000000000..ec2d68876a --- /dev/null +++ b/src/cli/commands/files/get.js @@ -0,0 +1,67 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') +var fs = require('fs') +const pathj = require('path') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + run: (path, outPath) => { + let dir + let filepath + let ws + + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!outPath) { + outPath = {} + dir = process.cwd() + } else { + if (outPath.slice(-1) !== '/') { + outPath += '/' + } + dir = outPath + } + + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.get(path, (err, data) => { + if (err) { + throw err + } + data.on('file', (data) => { + if (data.path.lastIndexOf('/') === -1) { + filepath = data.path + if (data.dir === false) { + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } else { + try { + fs.mkdirSync(pathj.join(dir, data.path)) + } catch (err) { + throw err + } + } + } else { + filepath = data.path.substring(0, data.path.lastIndexOf('/') + 1) + try { + fs.mkdirSync(pathj.join(dir, filepath)) + } catch (err) { + throw err + } + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } + }) + }) + }) + } +}) diff --git a/src/core/ipfs/files.js b/src/core/ipfs/files.js index b3a0cf705a..07e7da4ad4 100644 --- a/src/core/ipfs/files.js +++ b/src/core/ipfs/files.js @@ -1,13 +1,57 @@ 'use strict' -const importer = require('ipfs-data-importing').import +const Importer = require('ipfs-unixfs-engine').importer +const Exporter = require('ipfs-unixfs-engine').exporter +const UnixFS = require('ipfs-unixfs') -module.exports = function libp2p (self) { +module.exports = function files (self) { return { - add: (path, options, callback) => { - options.path = path - options.dagService = self._dagS - importer(options, callback) + add: (arr, callback) => { + if (typeof arr === 'function') { + callback = arr + arr = undefined + } + if (callback === undefined) { + callback = function noop () {} + } + if (arr === undefined) { + return new Importer(self._dagS) + } + + const i = new Importer(self._dagS) + const res = [] + + i.on('data', (info) => { + res.push(info) + }) + + i.once('end', () => { + callback(null, res) + }) + + arr.forEach((tuple) => { + i.write(tuple) + }) + + i.end() + }, + cat: (hash, callback) => { + self._dagS.get(hash, (err, fetchedNode) => { + if (err) { + return callback(err, null) + } + const data = UnixFS.unmarshal(fetchedNode.data) + if (data.type === 'directory') { + callback('This dag node is a directory', null) + } else { + const exportEvent = Exporter(hash, self._dagS) + callback(null, exportEvent) + } + }) + }, + get: (hash, callback) => { + var exportFile = Exporter(hash, self._dagS) + callback(null, exportFile) } } } diff --git a/src/core/ipfs/init.js b/src/core/ipfs/init.js index 1848e348c1..f69ba2fde9 100644 --- a/src/core/ipfs/init.js +++ b/src/core/ipfs/init.js @@ -4,6 +4,10 @@ const peerId = require('peer-id') const BlockService = require('ipfs-block-service') const DagService = require('ipfs-merkle-dag').DAGService const path = require('path') +const glob = require('glob') +const async = require('async') +const Readable = require('stream').Readable +const fs = require('fs') module.exports = function init (self) { return (opts, callback) => { @@ -67,15 +71,45 @@ module.exports = function init (self) { return doneImport(null) } - const importer = require('ipfs-data-importing') + const Importer = require('ipfs-unixfs-engine').importer const blocks = new BlockService(self._repo) const dag = new DagService(blocks) const initDocsPath = path.join(__dirname, '../../init-files/init-docs') - importer.import(initDocsPath, dag, { - recursive: true - }, doneImport) + const i = new Importer(dag) + i.resume() + + glob(path.join(initDocsPath, '/**/*'), (err, res) => { + if (err) { + throw err + } + const index = __dirname.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const buffered = fs.readFileSync(element) + const rs = new Readable() + rs.push(buffered) + rs.push(null) + const filePair = {path: addPath, stream: rs} + i.write(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + i.end() + return + }) + }) + + i.once('end', () => { + doneImport(null) + }) function doneImport (err, stat) { if (err) { return callback(err) } diff --git a/test/cli-tests/test-commands.js b/test/cli-tests/test-commands.js index aa8da00762..efb656d58a 100644 --- a/test/cli-tests/test-commands.js +++ b/test/cli-tests/test-commands.js @@ -10,7 +10,7 @@ describe('commands', () => { .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout.length).to.equal(45) + expect(stdout.length).to.equal(47) done() }) }) diff --git a/test/core-tests/test-files.js b/test/core-tests/test-files.js new file mode 100644 index 0000000000..4dd22d2777 --- /dev/null +++ b/test/core-tests/test-files.js @@ -0,0 +1,64 @@ +/* eslint-env mocha */ +'use strict' + +const bl = require('bl') +const expect = require('chai').expect +const Readable = require('stream').Readable +const bs58 = require('bs58') + +const IPFS = require('../../src/core') + +describe('files', () => { + let ipfs + + before((done) => { + ipfs = new IPFS(require('./repo-path')) + ipfs.load(done) + }) + + it('add', (done) => { + const buffered = new Buffer('some data') + const rs = new Readable() + rs.push(buffered) + rs.push(null) + const arr = [] + const filePair = {path: 'data.txt', stream: rs} + arr.push(filePair) + ipfs.files.add(arr, (err, res) => { + expect(err).to.not.exist + expect(res[0].path).to.equal('data.txt') + expect(res[0].size).to.equal(17) + expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + done() + }) + }) + + it('cat', (done) => { + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.cat(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) + + it('get', (done) => { + // TODO create non-trival get test + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.get(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) +}) diff --git a/test/core-tests/test-init-node.js b/test/core-tests/test-init-node.js index 1e05d61b5e..ee65c3e83e 100644 --- a/test/core-tests/test-init-node.js +++ b/test/core-tests/test-init-node.js @@ -24,14 +24,14 @@ describe('init (Node.js specific)', function () { it('init docs are written', (done) => { ipfs.init({ bits: 64 }, (err) => { expect(err).to.not.exist - - // Check for default assets var multihash = new Buffer('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') - ipfs.object.get(multihash, {}, (err, node) => { - expect(err).to.not.exist - expect(node.links).to.exist - done() - }) + setTimeout(() => { + ipfs.object.get(multihash, {}, (err, node) => { + expect(err).to.not.exist + expect(node.links).to.exist + done() + }) + }, 1000) }) }) diff --git a/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data new file mode 100644 index 0000000000..2965d1c457 --- /dev/null +++ b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data @@ -0,0 +1,3 @@ + + hello world + \ No newline at end of file From 8df6248585178657c876b0d17c16762fca1a3942 Mon Sep 17 00:00:00 2001 From: nginnever Date: Sun, 8 May 2016 10:00:38 -0700 Subject: [PATCH 2/6] updated get command / libp2p --- package.json | 6 +- src/cli/commands/files/cat.js | 1 - src/cli/commands/files/get.js | 126 ++++++++++++++++++++++------------ 3 files changed, 87 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index a1e79eeac7..b4ee6f04dc 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { "aegir": "^3.0.1", - "async": "^2.0.0-rc.3", + "async": "^1.5.2", "buffer-loader": "0.0.1", "chai": "^3.5.0", "expose-loader": "^0.7.1", @@ -70,10 +70,12 @@ "ipfs-repo": "^0.8.0", "ipfs-unixfs-engine": "^0.6.1", "joi": "^8.0.5", - "libp2p-ipfs": "^0.3.5", + "libp2p-ipfs": "^0.3.3", + "libp2p-swarm": "^0.12.5", "lodash.get": "^4.2.1", "lodash.set": "^4.1.0", "multiaddr": "^1.4.1", + "path-exists": "^3.0.0", "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js index 52dc3c1dbe..fe61d61316 100644 --- a/src/cli/commands/files/cat.js +++ b/src/cli/commands/files/cat.js @@ -31,7 +31,6 @@ module.exports = Command.extend({ data.stream.pipe(process.stdout) }) } - return }) }) } diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js index ec2d68876a..91a8308e5c 100644 --- a/src/cli/commands/files/get.js +++ b/src/cli/commands/files/get.js @@ -6,61 +6,101 @@ const utils = require('../../utils') const log = debug('cli:files') log.error = debug('cli:files:error') var fs = require('fs') -const pathj = require('path') +const path = require('path') +const pathExists = require('path-exists') +const async = require('async') -module.exports = Command.extend({ - desc: 'Download IPFS objects', - - run: (path, outPath) => { - let dir - let filepath - let ws - - if (!path) { - throw new Error("Argument 'path' is required") +function checkArgs (hash, outPath) { + if (!hash) { + throw new Error("Argument 'path' is required") + } + // format the output directory + if (!outPath) { + var cwd = process.cwd() + return cwd + } else { + if (!outPath.endsWith('/')) { + outPath += '/' } - if (!outPath) { - outPath = {} - dir = process.cwd() - } else { - if (outPath.slice(-1) !== '/') { - outPath += '/' + if (!outPath.startsWith('/')) { + outPath = path.join('/', outPath) + } + var directory = outPath + return directory + } +} + +function getFiles (result, dir) { + var filePath + result.on('file', (file) => { + // Check to see if the result is in a directory + if (file.path.lastIndexOf('/') === -1) { + filePath = file.path + // Check to see if the result is a directory + if (file.dir === false) { + const ws = fs.createWriteStream(path.join(dir, file.path)) + file.stream.pipe(ws) + } else { + // Check to see if the directory has already been created + pathExists(path.join(dir, file.path)).then(exists => { + if (!exists) { + fs.mkdir(path.join(dir, file.path), (err) => { + if (err) { + throw err + } + }) + } + }) } - dir = outPath + } else { + // Check to see if the directory has already been created + filePath = file.path.substring(0, file.path.lastIndexOf('/') + 1) + pathExists(path.join(dir, filePath)).then(exists => { + // Create a directory for the incoming files + if (!exists) { + async.waterfall([ + (cb) => { + fs.mkdir(path.join(dir, filePath), (err) => { + if (err) { + cb(err) + } + cb(null) + }) + }, + (cb) => { + const ws = fs.createWriteStream(path.join(dir, file.path)) + file.stream.pipe(ws) + cb(null) + } + ], (err) => { + if (err) { + throw err + } + }) + } + // Just write the file + const ws = fs.createWriteStream(path.join(dir, file.path)) + file.stream.pipe(ws) + }) } + }) +} + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + run: (hash, outPath) => { + const dir = checkArgs(hash, outPath) utils.getIPFS((err, ipfs) => { if (err) { throw err } - ipfs.files.get(path, (err, data) => { + ipfs.files.get(hash, (err, result) => { if (err) { throw err } - data.on('file', (data) => { - if (data.path.lastIndexOf('/') === -1) { - filepath = data.path - if (data.dir === false) { - ws = fs.createWriteStream(pathj.join(dir, data.path)) - data.stream.pipe(ws) - } else { - try { - fs.mkdirSync(pathj.join(dir, data.path)) - } catch (err) { - throw err - } - } - } else { - filepath = data.path.substring(0, data.path.lastIndexOf('/') + 1) - try { - fs.mkdirSync(pathj.join(dir, filepath)) - } catch (err) { - throw err - } - ws = fs.createWriteStream(pathj.join(dir, data.path)) - data.stream.pipe(ws) - } - }) + getFiles(result, dir) }) }) } From aae2259777bc01a4b453286f1d8bc318f17a9b57 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Mon, 9 May 2016 13:42:50 +0200 Subject: [PATCH 3/6] code review fixes --- package.json | 8 ++-- src/cli/commands/files/add.js | 15 +++----- src/cli/commands/files/get.js | 72 ++++++++++++++--------------------- src/core/ipfs/init.js | 16 ++++---- src/core/ipfs/libp2p.js | 6 +-- test/cli-tests/test-block.js | 1 - 6 files changed, 46 insertions(+), 72 deletions(-) diff --git a/package.json b/package.json index b4ee6f04dc..98114eaaee 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { "aegir": "^3.0.1", - "async": "^1.5.2", + "async": "^2.0.0-rc.4", "buffer-loader": "0.0.1", "chai": "^3.5.0", "expose-loader": "^0.7.1", @@ -64,7 +64,7 @@ "hapi": "^13.3.0", "ipfs-api": "^3.0.2", "ipfs-block": "^0.3.0", - "ipfs-block-service": "^0.3.0", + "ipfs-block-service": "^0.4.0", "ipfs-merkle-dag": "^0.5.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", @@ -79,7 +79,7 @@ "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", - "readable-stream": "^1.1.13", + "readable-stream": "1.1.13", "ronin": "^0.3.11", "temp": "^0.8.3" }, @@ -115,4 +115,4 @@ "kumavis ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 27ec59ee8c..cd33932112 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -31,7 +31,6 @@ function checkPath (inPath, recursive) { } if (inPath === '.' && recursive === true) { inPath = process.cwd() - return inPath } else if (inPath === '.' && recursive === false) { s = fs.statSync(process.cwd()) if (s.isDirectory()) { @@ -76,15 +75,13 @@ module.exports = Command.extend({ if (res.length !== 0) { const index = inPath.lastIndexOf('/') async.eachLimit(res, 10, (element, callback) => { - const addPath = element.substring(index + 1, element.length) - if (fs.statSync(element).isDirectory()) { - callback() - } else { - rs = fs.createReadStream(element) - filePair = {path: addPath, stream: rs} - i.write(filePair) - callback() + if (!fs.statSync(element).isDirectory()) { + i.write({ + path: element.substring(index + 1, element.length), + stream: fs.createReadStream(element) + }) } + callback() }, (err) => { if (err) { throw err diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js index 91a8308e5c..69b11fc01f 100644 --- a/src/cli/commands/files/get.js +++ b/src/cli/commands/files/get.js @@ -8,7 +8,6 @@ log.error = debug('cli:files:error') var fs = require('fs') const path = require('path') const pathExists = require('path-exists') -const async = require('async') function checkArgs (hash, outPath) { if (!hash) { @@ -30,60 +29,45 @@ function checkArgs (hash, outPath) { } } -function getFiles (result, dir) { - var filePath - result.on('file', (file) => { +function ensureDir (dir, cb) { + pathExists(dir) + .then((exists) => { + if (!exists) { + fs.mkdir(dir, cb) + } else { + cb() + } + }) + .catch(cb) +} + +function fileHandler (result, dir) { + return function onFile (file) { // Check to see if the result is in a directory if (file.path.lastIndexOf('/') === -1) { - filePath = file.path + const dirPath = path.join(dir, file.path) // Check to see if the result is a directory if (file.dir === false) { - const ws = fs.createWriteStream(path.join(dir, file.path)) - file.stream.pipe(ws) + file.stream.pipe(fs.createWriteStream(dirPath)) } else { - // Check to see if the directory has already been created - pathExists(path.join(dir, file.path)).then(exists => { - if (!exists) { - fs.mkdir(path.join(dir, file.path), (err) => { - if (err) { - throw err - } - }) + ensureDir(dirPath, (err) => { + if (err) { + throw err } }) } } else { - // Check to see if the directory has already been created - filePath = file.path.substring(0, file.path.lastIndexOf('/') + 1) - pathExists(path.join(dir, filePath)).then(exists => { - // Create a directory for the incoming files - if (!exists) { - async.waterfall([ - (cb) => { - fs.mkdir(path.join(dir, filePath), (err) => { - if (err) { - cb(err) - } - cb(null) - }) - }, - (cb) => { - const ws = fs.createWriteStream(path.join(dir, file.path)) - file.stream.pipe(ws) - cb(null) - } - ], (err) => { - if (err) { - throw err - } - }) + const filePath = file.path.substring(0, file.path.lastIndexOf('/') + 1) + const dirPath = path.join(dir, filePath) + ensureDir(dirPath, (err) => { + if (err) { + throw err } - // Just write the file - const ws = fs.createWriteStream(path.join(dir, file.path)) - file.stream.pipe(ws) + + file.stream.pipe(fs.createWriteStream(dirPath)) }) } - }) + } } module.exports = Command.extend({ @@ -100,7 +84,7 @@ module.exports = Command.extend({ if (err) { throw err } - getFiles(result, dir) + result.on('file', fileHandler(result, dir)) }) }) } diff --git a/src/core/ipfs/init.js b/src/core/ipfs/init.js index f69ba2fde9..2d8ee2e117 100644 --- a/src/core/ipfs/init.js +++ b/src/core/ipfs/init.js @@ -8,6 +8,7 @@ const glob = require('glob') const async = require('async') const Readable = require('stream').Readable const fs = require('fs') +const Importer = require('ipfs-unixfs-engine').importer module.exports = function init (self) { return (opts, callback) => { @@ -71,7 +72,6 @@ module.exports = function init (self) { return doneImport(null) } - const Importer = require('ipfs-unixfs-engine').importer const blocks = new BlockService(self._repo) const dag = new DagService(blocks) @@ -87,23 +87,19 @@ module.exports = function init (self) { const index = __dirname.lastIndexOf('/') async.eachLimit(res, 10, (element, callback) => { const addPath = element.substring(index + 1, element.length) - if (fs.statSync(element).isDirectory()) { - callback() - } else { - const buffered = fs.readFileSync(element) + if (!fs.statSync(element).isDirectory()) { const rs = new Readable() - rs.push(buffered) + rs.push(fs.readFileSync(element)) rs.push(null) const filePair = {path: addPath, stream: rs} i.write(filePair) - callback() } + callback() }, (err) => { if (err) { throw err } i.end() - return }) }) @@ -112,7 +108,9 @@ module.exports = function init (self) { }) function doneImport (err, stat) { - if (err) { return callback(err) } + if (err) { + return callback(err) + } // All finished! callback(null, true) diff --git a/src/core/ipfs/libp2p.js b/src/core/ipfs/libp2p.js index 3310d938a7..7548438397 100644 --- a/src/core/ipfs/libp2p.js +++ b/src/core/ipfs/libp2p.js @@ -20,11 +20,7 @@ module.exports = function libp2p (self) { }) }, stop: (callback) => { - try { - self._libp2pNode.swarm.close(callback) - } catch (err) { - console.log('It is fine :)') - } + self._libp2pNode.swarm.close(callback) }, swarm: { peers: (callback) => { diff --git a/test/cli-tests/test-block.js b/test/cli-tests/test-block.js index 970cd2cc7e..ea5cc15d40 100644 --- a/test/cli-tests/test-block.js +++ b/test/cli-tests/test-block.js @@ -71,7 +71,6 @@ describe('block', () => { describe('api running', () => { let httpAPI before((done) => { - console.log('repoPath ->', repoPath) httpAPI = new HttpAPI(repoPath) httpAPI.start((err) => { expect(err).to.not.exist From 46d9e6af3f6480702860e795f7016ab76e6c06ed Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Mon, 9 May 2016 14:09:20 +0200 Subject: [PATCH 4/6] fix(files:add): simplify checkPath --- src/cli/commands/files/add.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index cd33932112..080ed22be3 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -24,19 +24,14 @@ function checkPath (inPath, recursive) { throw new Error('Error: Argument \'path\' is required') } - var s = fs.statSync(inPath) - - if (s.isDirectory() && recursive === false) { - throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') - } - if (inPath === '.' && recursive === true) { + if (inPath === '.') { inPath = process.cwd() - } else if (inPath === '.' && recursive === false) { - s = fs.statSync(process.cwd()) - if (s.isDirectory()) { - throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') - } } + + if (fs.statSync(inPath).isDirectory() && recursive === false) { + throw new Error(`Error: ${inPath} is a directory, use the '-r' flag to specify directories`) + } + return inPath } From 7f89bfb494354956ac69ed7e01b8b0657fbd7082 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Mon, 9 May 2016 14:11:20 +0200 Subject: [PATCH 5/6] fix(files:get): simplify checkArgs --- src/cli/commands/files/get.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js index 69b11fc01f..31e8d17ad8 100644 --- a/src/cli/commands/files/get.js +++ b/src/cli/commands/files/get.js @@ -15,18 +15,18 @@ function checkArgs (hash, outPath) { } // format the output directory if (!outPath) { - var cwd = process.cwd() - return cwd - } else { - if (!outPath.endsWith('/')) { - outPath += '/' - } - if (!outPath.startsWith('/')) { - outPath = path.join('/', outPath) - } - var directory = outPath - return directory + return process.cwd() + } + + if (!outPath.endsWith('/')) { + outPath += '/' } + + if (!outPath.startsWith('/')) { + outPath = path.join('/', outPath) + } + + return outPath } function ensureDir (dir, cb) { From 56f965d32412e2f8ddd6a2cb864b8693717e1f5c Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Mon, 9 May 2016 14:13:49 +0200 Subject: [PATCH 6/6] test(http-api): give socket closing more time --- test/http-api-tests/test-swarm.js | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/test/http-api-tests/test-swarm.js b/test/http-api-tests/test-swarm.js index 8b460e2ca7..73610825a9 100644 --- a/test/http-api-tests/test-swarm.js +++ b/test/http-api-tests/test-swarm.js @@ -31,20 +31,9 @@ module.exports = (httpAPI) => { }) after((done) => { - // cause CI takes forever - var closed = false setTimeout(() => { - if (!closed) { - closed = true - done() - } - }, 10000) - ipfs.libp2p.stop(() => { - if (!closed) { - closed = true - done() - } - }) + ipfs.libp2p.stop(done) + }, 1000) }) it('gets the api obj', () => { @@ -129,20 +118,9 @@ module.exports = (httpAPI) => { }) after((done) => { - // cause CI takes forever - var closed = false setTimeout(() => { - if (!closed) { - closed = true - done() - } - }, 10000) - ipfs.libp2p.stop(() => { - if (!closed) { - closed = true - done() - } - }) + ipfs.libp2p.stop(done) + }, 1000) }) it('start IPFS API ctl', (done) => {