diff --git a/package.json b/package.json index 98bce8b23f..98114eaaee 100644 --- a/package.json +++ b/package.json @@ -38,45 +38,48 @@ "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { "aegir": "^3.0.1", - "async": "^2.0.0-rc.3", + "async": "^2.0.0-rc.4", "buffer-loader": "0.0.1", "chai": "^3.5.0", "expose-loader": "^0.7.1", "form-data": "^1.0.0-rc3", "idb-plus-blob-store": "^1.1.2", - "lodash": "^4.11.1", - "mocha": "^2.3.4", + "lodash": "^4.11.2", + "mocha": "^2.4.5", "ncp": "^2.0.0", "nexpect": "^0.5.0", "pre-commit": "^1.1.2", - "rimraf": "^2.4.4", + "rimraf": "^2.5.2", "stream-to-promise": "^1.1.0", "transform-loader": "^0.2.3" }, "dependencies": { "babel-runtime": "^6.6.1", "bl": "^1.1.2", - "boom": "^3.1.1", + "boom": "^3.1.2", "bs58": "^3.0.0", "debug": "^2.2.0", "fs-blob-store": "^5.2.1", "glob": "^7.0.3", "hapi": "^13.3.0", - "ipfs-api": "^3.0.1", + "ipfs-api": "^3.0.2", "ipfs-block": "^0.3.0", - "ipfs-block-service": "^0.3.0", - "ipfs-data-importing": "^0.3.3", + "ipfs-block-service": "^0.4.0", "ipfs-merkle-dag": "^0.5.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", - "joi": "^8.0.2", + "ipfs-unixfs-engine": "^0.6.1", + "joi": "^8.0.5", "libp2p-ipfs": "^0.3.3", + "libp2p-swarm": "^0.12.5", "lodash.get": "^4.2.1", - "lodash.set": "^4.0.0", - "multiaddr": "^1.3.0", + "lodash.set": "^4.1.0", + "multiaddr": "^1.4.1", + "path-exists": "^3.0.0", "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", + "readable-stream": "1.1.13", "ronin": "^0.3.11", "temp": "^0.8.3" }, @@ -112,4 +115,4 @@ "kumavis ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 5a0fd79138..080ed22be3 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -1,11 +1,39 @@ 'use strict' const Command = require('ronin').Command -const IPFS = require('../../../core') +const utils = require('../../utils') const debug = require('debug') const log = debug('cli:version') log.error = debug('cli:version:error') const bs58 = require('bs58') +const fs = require('fs') +const async = require('async') +const path = require('path') +const glob = require('glob') + +function checkPath (inPath, recursive) { + // This function is to check for the following possible inputs + // 1) "." add the cwd but throw error for no recursion flag + // 2) "." -r return the cwd + // 3) "/some/path" but throw error for no recursion + // 4) "/some/path" -r + // 5) No path, throw err + // 6) filename.type return the cwd + filename + + if (!inPath) { + throw new Error('Error: Argument \'path\' is required') + } + + if (inPath === '.') { + inPath = process.cwd() + } + + if (fs.statSync(inPath).isDirectory() && recursive === false) { + throw new Error(`Error: ${inPath} is a directory, use the '-r' flag to specify directories`) + } + + return inPath +} module.exports = Command.extend({ desc: 'Add a file to IPFS using the UnixFS data format', @@ -18,16 +46,51 @@ module.exports = Command.extend({ } }, - run: (recursive, path) => { - var node = new IPFS() - path = process.cwd() + '/' + path - node.files.add(path, { - recursive: recursive - }, (err, stats) => { + run: (recursive, inPath) => { + let rs + + inPath = checkPath(inPath, recursive) + + glob(path.join(inPath, '/**/*'), (err, res) => { if (err) { - return console.log(err) + throw err } - console.log('added', bs58.encode(stats.Hash).toString(), stats.Name) + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + const i = ipfs.files.add() + var filePair + i.on('data', (file) => { + console.log('added', bs58.encode(file.multihash).toString(), file.path) + }) + i.once('end', () => { + return + }) + if (res.length !== 0) { + const index = inPath.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + if (!fs.statSync(element).isDirectory()) { + i.write({ + path: element.substring(index + 1, element.length), + stream: fs.createReadStream(element) + }) + } + callback() + }, (err) => { + if (err) { + throw err + } + i.end() + }) + } else { + rs = fs.createReadStream(inPath) + inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length) + filePair = {path: inPath, stream: rs} + i.write(filePair) + i.end() + } + }) }) } }) diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js new file mode 100644 index 0000000000..fe61d61316 --- /dev/null +++ b/src/cli/commands/files/cat.js @@ -0,0 +1,37 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + } + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.cat(path, (err, res) => { + if (err) { + throw (err) + } + if (res) { + res.on('file', (data) => { + data.stream.pipe(process.stdout) + }) + } + }) + }) + } +}) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js new file mode 100644 index 0000000000..31e8d17ad8 --- /dev/null +++ b/src/cli/commands/files/get.js @@ -0,0 +1,91 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') +var fs = require('fs') +const path = require('path') +const pathExists = require('path-exists') + +function checkArgs (hash, outPath) { + if (!hash) { + throw new Error("Argument 'path' is required") + } + // format the output directory + if (!outPath) { + return process.cwd() + } + + if (!outPath.endsWith('/')) { + outPath += '/' + } + + if (!outPath.startsWith('/')) { + outPath = path.join('/', outPath) + } + + return outPath +} + +function ensureDir (dir, cb) { + pathExists(dir) + .then((exists) => { + if (!exists) { + fs.mkdir(dir, cb) + } else { + cb() + } + }) + .catch(cb) +} + +function fileHandler (result, dir) { + return function onFile (file) { + // Check to see if the result is in a directory + if (file.path.lastIndexOf('/') === -1) { + const dirPath = path.join(dir, file.path) + // Check to see if the result is a directory + if (file.dir === false) { + file.stream.pipe(fs.createWriteStream(dirPath)) + } else { + ensureDir(dirPath, (err) => { + if (err) { + throw err + } + }) + } + } else { + const filePath = file.path.substring(0, file.path.lastIndexOf('/') + 1) + const dirPath = path.join(dir, filePath) + ensureDir(dirPath, (err) => { + if (err) { + throw err + } + + file.stream.pipe(fs.createWriteStream(dirPath)) + }) + } + } +} + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + run: (hash, outPath) => { + const dir = checkArgs(hash, outPath) + + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.get(hash, (err, result) => { + if (err) { + throw err + } + result.on('file', fileHandler(result, dir)) + }) + }) + } +}) diff --git a/src/core/ipfs/files.js b/src/core/ipfs/files.js index b3a0cf705a..07e7da4ad4 100644 --- a/src/core/ipfs/files.js +++ b/src/core/ipfs/files.js @@ -1,13 +1,57 @@ 'use strict' -const importer = require('ipfs-data-importing').import +const Importer = require('ipfs-unixfs-engine').importer +const Exporter = require('ipfs-unixfs-engine').exporter +const UnixFS = require('ipfs-unixfs') -module.exports = function libp2p (self) { +module.exports = function files (self) { return { - add: (path, options, callback) => { - options.path = path - options.dagService = self._dagS - importer(options, callback) + add: (arr, callback) => { + if (typeof arr === 'function') { + callback = arr + arr = undefined + } + if (callback === undefined) { + callback = function noop () {} + } + if (arr === undefined) { + return new Importer(self._dagS) + } + + const i = new Importer(self._dagS) + const res = [] + + i.on('data', (info) => { + res.push(info) + }) + + i.once('end', () => { + callback(null, res) + }) + + arr.forEach((tuple) => { + i.write(tuple) + }) + + i.end() + }, + cat: (hash, callback) => { + self._dagS.get(hash, (err, fetchedNode) => { + if (err) { + return callback(err, null) + } + const data = UnixFS.unmarshal(fetchedNode.data) + if (data.type === 'directory') { + callback('This dag node is a directory', null) + } else { + const exportEvent = Exporter(hash, self._dagS) + callback(null, exportEvent) + } + }) + }, + get: (hash, callback) => { + var exportFile = Exporter(hash, self._dagS) + callback(null, exportFile) } } } diff --git a/src/core/ipfs/init.js b/src/core/ipfs/init.js index 1848e348c1..2d8ee2e117 100644 --- a/src/core/ipfs/init.js +++ b/src/core/ipfs/init.js @@ -4,6 +4,11 @@ const peerId = require('peer-id') const BlockService = require('ipfs-block-service') const DagService = require('ipfs-merkle-dag').DAGService const path = require('path') +const glob = require('glob') +const async = require('async') +const Readable = require('stream').Readable +const fs = require('fs') +const Importer = require('ipfs-unixfs-engine').importer module.exports = function init (self) { return (opts, callback) => { @@ -67,18 +72,45 @@ module.exports = function init (self) { return doneImport(null) } - const importer = require('ipfs-data-importing') const blocks = new BlockService(self._repo) const dag = new DagService(blocks) const initDocsPath = path.join(__dirname, '../../init-files/init-docs') - importer.import(initDocsPath, dag, { - recursive: true - }, doneImport) + const i = new Importer(dag) + i.resume() + + glob(path.join(initDocsPath, '/**/*'), (err, res) => { + if (err) { + throw err + } + const index = __dirname.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + const addPath = element.substring(index + 1, element.length) + if (!fs.statSync(element).isDirectory()) { + const rs = new Readable() + rs.push(fs.readFileSync(element)) + rs.push(null) + const filePair = {path: addPath, stream: rs} + i.write(filePair) + } + callback() + }, (err) => { + if (err) { + throw err + } + i.end() + }) + }) + + i.once('end', () => { + doneImport(null) + }) function doneImport (err, stat) { - if (err) { return callback(err) } + if (err) { + return callback(err) + } // All finished! callback(null, true) diff --git a/src/core/ipfs/libp2p.js b/src/core/ipfs/libp2p.js index 3310d938a7..7548438397 100644 --- a/src/core/ipfs/libp2p.js +++ b/src/core/ipfs/libp2p.js @@ -20,11 +20,7 @@ module.exports = function libp2p (self) { }) }, stop: (callback) => { - try { - self._libp2pNode.swarm.close(callback) - } catch (err) { - console.log('It is fine :)') - } + self._libp2pNode.swarm.close(callback) }, swarm: { peers: (callback) => { diff --git a/test/cli-tests/test-block.js b/test/cli-tests/test-block.js index 970cd2cc7e..ea5cc15d40 100644 --- a/test/cli-tests/test-block.js +++ b/test/cli-tests/test-block.js @@ -71,7 +71,6 @@ describe('block', () => { describe('api running', () => { let httpAPI before((done) => { - console.log('repoPath ->', repoPath) httpAPI = new HttpAPI(repoPath) httpAPI.start((err) => { expect(err).to.not.exist diff --git a/test/cli-tests/test-commands.js b/test/cli-tests/test-commands.js index aa8da00762..efb656d58a 100644 --- a/test/cli-tests/test-commands.js +++ b/test/cli-tests/test-commands.js @@ -10,7 +10,7 @@ describe('commands', () => { .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout.length).to.equal(45) + expect(stdout.length).to.equal(47) done() }) }) diff --git a/test/core-tests/test-files.js b/test/core-tests/test-files.js new file mode 100644 index 0000000000..4dd22d2777 --- /dev/null +++ b/test/core-tests/test-files.js @@ -0,0 +1,64 @@ +/* eslint-env mocha */ +'use strict' + +const bl = require('bl') +const expect = require('chai').expect +const Readable = require('stream').Readable +const bs58 = require('bs58') + +const IPFS = require('../../src/core') + +describe('files', () => { + let ipfs + + before((done) => { + ipfs = new IPFS(require('./repo-path')) + ipfs.load(done) + }) + + it('add', (done) => { + const buffered = new Buffer('some data') + const rs = new Readable() + rs.push(buffered) + rs.push(null) + const arr = [] + const filePair = {path: 'data.txt', stream: rs} + arr.push(filePair) + ipfs.files.add(arr, (err, res) => { + expect(err).to.not.exist + expect(res[0].path).to.equal('data.txt') + expect(res[0].size).to.equal(17) + expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + done() + }) + }) + + it('cat', (done) => { + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.cat(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) + + it('get', (done) => { + // TODO create non-trival get test + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.get(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) +}) diff --git a/test/core-tests/test-init-node.js b/test/core-tests/test-init-node.js index 1e05d61b5e..ee65c3e83e 100644 --- a/test/core-tests/test-init-node.js +++ b/test/core-tests/test-init-node.js @@ -24,14 +24,14 @@ describe('init (Node.js specific)', function () { it('init docs are written', (done) => { ipfs.init({ bits: 64 }, (err) => { expect(err).to.not.exist - - // Check for default assets var multihash = new Buffer('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') - ipfs.object.get(multihash, {}, (err, node) => { - expect(err).to.not.exist - expect(node.links).to.exist - done() - }) + setTimeout(() => { + ipfs.object.get(multihash, {}, (err, node) => { + expect(err).to.not.exist + expect(node.links).to.exist + done() + }) + }, 1000) }) }) diff --git a/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data new file mode 100644 index 0000000000..2965d1c457 --- /dev/null +++ b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data @@ -0,0 +1,3 @@ + + hello world + \ No newline at end of file diff --git a/test/http-api-tests/test-swarm.js b/test/http-api-tests/test-swarm.js index 8b460e2ca7..73610825a9 100644 --- a/test/http-api-tests/test-swarm.js +++ b/test/http-api-tests/test-swarm.js @@ -31,20 +31,9 @@ module.exports = (httpAPI) => { }) after((done) => { - // cause CI takes forever - var closed = false setTimeout(() => { - if (!closed) { - closed = true - done() - } - }, 10000) - ipfs.libp2p.stop(() => { - if (!closed) { - closed = true - done() - } - }) + ipfs.libp2p.stop(done) + }, 1000) }) it('gets the api obj', () => { @@ -129,20 +118,9 @@ module.exports = (httpAPI) => { }) after((done) => { - // cause CI takes forever - var closed = false setTimeout(() => { - if (!closed) { - closed = true - done() - } - }, 10000) - ipfs.libp2p.stop(() => { - if (!closed) { - closed = true - done() - } - }) + ipfs.libp2p.stop(done) + }, 1000) }) it('start IPFS API ctl', (done) => {