files add, cat, get core + cli

package.json

@@ -37,46 +37,47 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^3.0.0",
+    "aegir": "^3.0.1",
     "async": "^2.0.0-rc.3",
     "buffer-loader": "0.0.1",
     "chai": "^3.5.0",
     "expose-loader": "^0.7.1",
     "form-data": "^1.0.0-rc3",
     "idb-plus-blob-store": "^1.1.2",
-    "lodash": "^4.11.1",
-    "mocha": "^2.3.4",
+    "lodash": "^4.11.2",
+    "mocha": "^2.4.5",
     "ncp": "^2.0.0",
     "nexpect": "^0.5.0",
     "pre-commit": "^1.1.2",
-    "rimraf": "^2.4.4",
+    "rimraf": "^2.5.2",
     "stream-to-promise": "^1.1.0",
     "transform-loader": "^0.2.3"
   },
   "dependencies": {
     "babel-runtime": "^6.6.1",
     "bl": "^1.1.2",
-    "boom": "^3.1.1",
+    "boom": "^3.1.2",
     "bs58": "^3.0.0",
     "debug": "^2.2.0",
     "fs-blob-store": "^5.2.1",
     "glob": "^7.0.3",
     "hapi": "^13.3.0",
-    "ipfs-api": "^3.0.1",
+    "ipfs-api": "^3.0.2",
     "ipfs-block": "^0.3.0",
     "ipfs-block-service": "^0.3.0",
-    "ipfs-data-importing": "^0.3.3",
     "ipfs-merkle-dag": "^0.5.0",
     "ipfs-multipart": "^0.1.0",
     "ipfs-repo": "^0.8.0",
-    "joi": "^8.0.2",
-    "libp2p-ipfs": "^0.3.3",
+    "ipfs-unixfs-engine": "^0.6.1",
+    "joi": "^8.0.5",
+    "libp2p-ipfs": "^0.3.5",
     "lodash.get": "^4.2.1",
-    "lodash.set": "^4.0.0",
-    "multiaddr": "^1.3.0",
+    "lodash.set": "^4.1.0",
+    "multiaddr": "^1.4.1",
     "peer-book": "0.1.0",
     "peer-id": "^0.6.6",
     "peer-info": "^0.6.2",
+    "readable-stream": "^1.1.13",
     "ronin": "^0.3.11",
     "temp": "^0.8.3"
   },

src/cli/commands/files/add.js

@@ -1,11 +1,45 @@
 'use strict'
 
 const Command = require('ronin').Command
-const IPFS = require('../../../core')
+const utils = require('../../utils')
 const debug = require('debug')
 const log = debug('cli:version')
 log.error = debug('cli:version:error')
 const bs58 = require('bs58')
+const fs = require('fs')
+const async = require('async')
+const path = require('path')
+const glob = require('glob')
+
+function checkPath (inPath, recursive) {
+  // This function is to check for the following possible inputs
+  // 1) "." add the cwd but throw error for no recursion flag
+  // 2) "." -r return the cwd
+  // 3) "/some/path" but throw error for no recursion
+  // 4) "/some/path" -r
+  // 5) No path, throw err
+  // 6) filename.type return the cwd + filename
+
+  if (!inPath) {
+    throw new Error('Error: Argument \'path\' is required')
+  }
+
+  var s = fs.statSync(inPath)
+
+  if (s.isDirectory() && recursive === false) {
+    throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories')
+  }
+  if (inPath === '.' && recursive === true) {
+    inPath = process.cwd()
+    return inPath
+  } else if (inPath === '.' && recursive === false) {
+    s = fs.statSync(process.cwd())
+    if (s.isDirectory()) {
+      throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories')
+    }
+  }
+  return inPath
+}
 
 module.exports = Command.extend({
   desc: 'Add a file to IPFS using the UnixFS data format',
@@ -18,16 +52,53 @@ module.exports = Command.extend({
     }
   },
 
-  run: (recursive, path) => {
-    var node = new IPFS()
-    path = process.cwd() + '/' + path
-    node.files.add(path, {
-      recursive: recursive
-    }, (err, stats) => {
+  run: (recursive, inPath) => {
+    let rs
+
+    inPath = checkPath(inPath, recursive)
+
+    glob(path.join(inPath, '/**/*'), (err, res) => {
       if (err) {
-        return console.log(err)
+        throw err
       }
-      console.log('added', bs58.encode(stats.Hash).toString(), stats.Name)
+      utils.getIPFS((err, ipfs) => {
+        if (err) {
+          throw err
+        }
+        const i = ipfs.files.add()
+        var filePair
+        i.on('data', (file) => {
+          console.log('added', bs58.encode(file.multihash).toString(), file.path)
+        })
+        i.once('end', () => {
+          return
+        })
+        if (res.length !== 0) {
+          const index = inPath.lastIndexOf('/')
+          async.eachLimit(res, 10, (element, callback) => {
+            const addPath = element.substring(index + 1, element.length)
+            if (fs.statSync(element).isDirectory()) {
+              callback()
+            } else {
+              rs = fs.createReadStream(element)
+              filePair = {path: addPath, stream: rs}
+              i.write(filePair)
+              callback()
+            }
+          }, (err) => {
+            if (err) {
+              throw err
+            }
+            i.end()
+          })
+        } else {
+          rs = fs.createReadStream(inPath)
+          inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length)
+          filePair = {path: inPath, stream: rs}
+          i.write(filePair)
+          i.end()
+        }
+      })
     })
   }
 })

src/cli/commands/files/cat.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const Command = require('ronin').Command
+const debug = require('debug')
+const utils = require('../../utils')
+const log = debug('cli:files')
+log.error = debug('cli:files:error')
+
+module.exports = Command.extend({
+  desc: 'Download IPFS objects',
+
+  options: {},
+
+  run: (path, options) => {
+    if (!path) {
+      throw new Error("Argument 'path' is required")
+    }
+    if (!options) {
+      options = {}
+    }
+    utils.getIPFS((err, ipfs) => {
+      if (err) {
+        throw err
+      }
+      ipfs.files.cat(path, (err, res) => {
+        if (err) {
+          throw (err)
+        }
+        if (res) {
+          res.on('file', (data) => {
+            data.stream.pipe(process.stdout)
+          })
+        }
+        return
+      })
+    })
+  }
+})

src/cli/commands/files/get.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const Command = require('ronin').Command
+const debug = require('debug')
+const utils = require('../../utils')
+const log = debug('cli:files')
+log.error = debug('cli:files:error')
+var fs = require('fs')
+const pathj = require('path')
+
+module.exports = Command.extend({
+  desc: 'Download IPFS objects',
+
+  run: (path, outPath) => {
+    let dir
+    let filepath
+    let ws
+
+    if (!path) {
+      throw new Error("Argument 'path' is required")
+    }
+    if (!outPath) {
+      outPath = {}
+      dir = process.cwd()
+    } else {
+      if (outPath.slice(-1) !== '/') {
+        outPath += '/'
+      }
+      dir = outPath
+    }
+
+    utils.getIPFS((err, ipfs) => {
+      if (err) {
+        throw err
+      }
+      ipfs.files.get(path, (err, data) => {
+        if (err) {
+          throw err
+        }
+        data.on('file', (data) => {
+          if (data.path.lastIndexOf('/') === -1) {
+            filepath = data.path
+            if (data.dir === false) {
+              ws = fs.createWriteStream(pathj.join(dir, data.path))
+              data.stream.pipe(ws)
+            } else {
+              try {
+                fs.mkdirSync(pathj.join(dir, data.path))
+              } catch (err) {
+                throw err
+              }
+            }
+          } else {
+            filepath = data.path.substring(0, data.path.lastIndexOf('/') + 1)
+            try {
+              fs.mkdirSync(pathj.join(dir, filepath))
+            } catch (err) {
+              throw err
+            }
+            ws = fs.createWriteStream(pathj.join(dir, data.path))
+            data.stream.pipe(ws)
+          }
+        })
+      })
+    })
+  }
+})

src/core/ipfs/files.js

@@ -1,13 +1,57 @@
 'use strict'
 
-const importer = require('ipfs-data-importing').import
+const Importer = require('ipfs-unixfs-engine').importer
+const Exporter = require('ipfs-unixfs-engine').exporter
+const UnixFS = require('ipfs-unixfs')
 
-module.exports = function libp2p (self) {
+module.exports = function files (self) {
   return {
-    add: (path, options, callback) => {
-      options.path = path
-      options.dagService = self._dagS
-      importer(options, callback)
+    add: (arr, callback) => {
+      if (typeof arr === 'function') {
+        callback = arr
+        arr = undefined
+      }
+      if (callback === undefined) {
+        callback = function noop () {}
+      }
+      if (arr === undefined) {
+        return new Importer(self._dagS)
+      }
+
+      const i = new Importer(self._dagS)
+      const res = []
+
+      i.on('data', (info) => {
+        res.push(info)
+      })
+
+      i.once('end', () => {
+        callback(null, res)
+      })
+
+      arr.forEach((tuple) => {
+        i.write(tuple)
+      })
+
+      i.end()
+    },
+    cat: (hash, callback) => {
+      self._dagS.get(hash, (err, fetchedNode) => {
+        if (err) {
+          return callback(err, null)
+        }
+        const data = UnixFS.unmarshal(fetchedNode.data)
+        if (data.type === 'directory') {
+          callback('This dag node is a directory', null)
+        } else {
+          const exportEvent = Exporter(hash, self._dagS)
+          callback(null, exportEvent)
+        }
+      })
+    },
+    get: (hash, callback) => {
+      var exportFile = Exporter(hash, self._dagS)
+      callback(null, exportFile)
     }
   }
 }

src/core/ipfs/init.js

@@ -4,6 +4,10 @@ const peerId = require('peer-id')
 const BlockService = require('ipfs-block-service')
 const DagService = require('ipfs-merkle-dag').DAGService
 const path = require('path')
+const glob = require('glob')
+const async = require('async')
+const Readable = require('stream').Readable
+const fs = require('fs')
 
 module.exports = function init (self) {
   return (opts, callback) => {
@@ -67,15 +71,45 @@ module.exports = function init (self) {
         return doneImport(null)
       }
 
-      const importer = require('ipfs-data-importing')
+      const Importer = require('ipfs-unixfs-engine').importer
       const blocks = new BlockService(self._repo)
       const dag = new DagService(blocks)
 
       const initDocsPath = path.join(__dirname, '../../init-files/init-docs')
 
-      importer.import(initDocsPath, dag, {
-        recursive: true
-      }, doneImport)
+      const i = new Importer(dag)
+      i.resume()
+
+      glob(path.join(initDocsPath, '/**/*'), (err, res) => {
+        if (err) {
+          throw err
+        }
+        const index = __dirname.lastIndexOf('/')
+        async.eachLimit(res, 10, (element, callback) => {
+          const addPath = element.substring(index + 1, element.length)
+          if (fs.statSync(element).isDirectory()) {
+            callback()
+          } else {
+            const buffered = fs.readFileSync(element)
+            const rs = new Readable()
+            rs.push(buffered)
+            rs.push(null)
+            const filePair = {path: addPath, stream: rs}
+            i.write(filePair)
+            callback()
+          }
+        }, (err) => {
+          if (err) {
+            throw err
+          }
+          i.end()
+          return
+        })
+      })
+
+      i.once('end', () => {
+        doneImport(null)
+      })
 
       function doneImport (err, stat) {
         if (err) { return callback(err) }