Pinning core, cli and http

package.json

@@ -64,6 +64,7 @@
     "bs58": "^3.0.0",
     "debug": "^2.2.0",
     "detect-node": "^2.0.3",
+    "fnv1a": "^1.0.1",
     "fs-blob-store": "^5.2.1",
     "glob": "^7.0.5",
     "hapi": "^14.0.0",

src/cli/commands/pin.js

@@ -0,0 +1,15 @@
+'use strict'
+
+module.exports = {
+  command: 'pin',
+
+  description: 'Pin and unpin objects to local storage.',
+
+  builder (yargs) {
+    return yargs
+      .commandDir('pin')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/pin/add.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const utils = require('../../utils')
+const bs58 = require('bs58')
+const debug = require('debug')
+const log = debug('cli:pin')
+log.error = debug('cli:pin:error')
+
+const onError = (err) => {
+  if (err) {
+    console.error(err)
+    throw err
+  }
+}
+
+module.exports = {
+  command: 'add <ipfs-path>',
+
+  describe: 'Pins objects to local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively pin the object linked to by the specified object(s).'
+    }
+  },
+
+  handler (argv) {
+    const path = argv['ipfs-path']
+    const recursive = argv.recursive
+    utils.getIPFS((err, ipfs) => {
+      onError(err)
+      // load persistent pin set from datastore
+      ipfs.pinner.load(() => {
+        const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/)
+        if (!matched) {
+          onError(new Error('invalid ipfs ref path'))
+        }
+        const split = matched[1].split('/')
+        const rootHash = split[0]
+        const key = new Buffer(bs58.decode(rootHash))
+        const links = split.slice(1, split.length)
+        const pathFn = (err, obj) => {
+          onError(err)
+          if (links.length) {
+            const linkName = links.shift()
+            const nextLink = obj.links.filter((link) => {
+              return (link.name === linkName)
+            })
+            if (!nextLink.length) {
+              onError(new Error(
+                'pin: no link named ' + linkName +
+                ' under ' + obj.toJSON().Hash
+              ))
+            }
+            const nextHash = nextLink[0].hash
+            ipfs.object.get(nextHash, pathFn)
+          } else {
+            ipfs.pinner.pin(obj, recursive, (err) => {
+              onError(err)
+              // save modified pin state to datastore
+              ipfs.pinner.flush((err, root) => {
+                onError(err)
+                const mode = recursive ? ' recursively' : ' directly'
+                console.log('pinned ' + obj.toJSON().Hash + mode)
+              })
+            })
+          }
+        }
+        ipfs.object.get(key, pathFn)
+      })
+    })
+  }
+}

src/cli/commands/pin/ls.js

@@ -0,0 +1,133 @@
+'use strict'
+
+const utils = require('../../utils')
+const bs58 = require('bs58')
+const debug = require('debug')
+const log = debug('cli:pin')
+log.error = debug('cli:pin:error')
+
+const onError = (err) => {
+  if (err) {
+    console.error(err)
+    throw err
+  }
+}
+
+module.exports = {
+  command: 'ls',
+
+  describe: 'List objects pinned to local storage.',
+
+  builder: {
+    path: {
+      type: 'string',
+      describe: 'List pinned state of specific <ipfs-path>.'
+    },
+    type: {
+      type: 'string',
+      alias: 't',
+      default: 'all',
+      describe: ('The type of pinned keys to list. ' +
+                 'Can be "direct", "indirect", "recursive", or "all".')
+    },
+    quiet: {
+      type: 'boolean',
+      alias: 'q',
+      default: false,
+      describe: 'Write just hashes of objects.'
+    }
+  },
+
+  handler: (argv) => {
+    const path = argv.path
+    const type = argv.type
+    const quiet = argv.quiet
+    utils.getIPFS((err, ipfs) => {
+      onError(err)
+      const types = ipfs.pinner.types
+      // load persistent pin set from datastore
+      ipfs.pinner.load(() => {
+        if (path) {
+          const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/)
+          if (!matched) {
+            onError(new Error('invalid ipfs ref path'))
+          }
+          const split = matched[1].split('/')
+          const rootHash = split[0]
+          const key = new Buffer(bs58.decode(rootHash))
+          const links = split.slice(1, split.length)
+          const pathFn = (err, obj) => {
+            onError(err)
+            if (links.length) {
+              const linkName = links.shift()
+              const nextLink = obj.links.filter((link) => {
+                return (link.name === linkName)
+              })
+              if (!nextLink.length) {
+                onError(new Error(
+                  'pin: no link named ' + linkName +
+                  ' under ' + obj.toJSON().Hash
+                ))
+              }
+              const nextHash = nextLink[0].hash
+              ipfs.object.get(nextHash, pathFn)
+            } else {
+              ipfs.pinner.isPinnedWithType(obj.multihash(), type, (err, pinned, reason) => {
+                onError(err)
+                if (!pinned) {
+                  onError(new Error('Path ' + path + ' is not pinned'))
+                }
+                if (reason !== types.direct &&
+                    reason !== types.recursive) {
+                  reason = 'indirect through ' + reason
+                }
+                console.log(obj.toJSON().Hash + (quiet ? '' : ' ' + reason))
+              })
+            }
+          }
+          ipfs.object.get(key, pathFn)
+        } else {
+          const printDirect = () => {
+            ipfs.pinner.directKeyStrings().forEach((key) => {
+              console.log(key + (quiet ? '' : ' direct'))
+            })
+          }
+          const printRecursive = () => {
+            ipfs.pinner.recursiveKeyStrings().forEach((key) => {
+              console.log(key + (quiet ? '' : ' recursive'))
+            })
+          }
+          const printIndirect = () => {
+            ipfs.pinner.getIndirectKeys((err, keys) => {
+              onError(err)
+              keys.forEach((key) => {
+                console.log(key + (quiet ? '' : ' indirect'))
+              })
+            })
+          }
+          switch (type) {
+            case types.direct:
+              printDirect()
+              break
+            case types.recursive:
+              printRecursive()
+              break
+            case types.indirect:
+              printIndirect()
+              break
+            case types.all:
+              printDirect()
+              printRecursive()
+              printIndirect()
+              break
+            default:
+              onError(new Error(
+                "Invalid type '" + type + "', " +
+                'must be one of {direct, indirect, recursive, all}'
+              ))
+          }
+        }
+      })
+    })
+  }
+}

src/cli/commands/pin/rm.js

@@ -0,0 +1,81 @@
+'use strict'
+
+const utils = require('../../utils')
+const bs58 = require('bs58')
+const debug = require('debug')
+const log = debug('cli:pin')
+log.error = debug('cli:pin:error')
+
+const onError = (err) => {
+  if (err) {
+    console.error(err)
+    throw err
+  }
+}
+
+module.exports = {
+  command: 'rm <ipfs-path>',
+
+  describe: 'Removes the pinned object from local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively unpin the objects linked to by the specified object(s).'
+    }
+  },
+
+  handler: (argv) => {
+    const path = argv['ipfs-path']
+    const recursive = argv.recursive
+    utils.getIPFS((err, ipfs) => {
+      onError(err)
+      // load persistent pin set from datastore
+      ipfs.pinner.load(() => {
+        const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/)
+        if (!matched) {
+          onError(new Error('invalid ipfs ref path'))
+        }
+        const split = matched[1].split('/')
+        const rootHash = split[0]
+        const key = new Buffer(bs58.decode(rootHash))
+        const links = split.slice(1, split.length)
+        const pathFn = (err, obj) => {
+          onError(err)
+          if (links.length) {
+            const linkName = links.shift()
+            const nextLink = obj.links.filter((link) => {
+              return (link.name === linkName)
+            })
+            if (!nextLink.length) {
+              onError(new Error(
+                'pin: no link named ' + linkName +
+                ' under ' + obj.toJSON().Hash
+              ))
+            }
+            const nextHash = nextLink[0].hash
+            ipfs.object.get(nextHash, pathFn)
+          } else {
+            ipfs.pinner.isPinned(obj.multihash(), (err, pinned, reason) => {
+              onError(err)
+              if (!pinned) {
+                onError(new Error('not pinned'))
+              }
+              ipfs.pinner.unpin(obj.multihash(), recursive, (err) => {
+                onError(err)
+                // save modified pin state to datastore
+                ipfs.pinner.flush((err, root) => {
+                  onError(err)
+                  console.log('unpinned ' + obj.toJSON().Hash)
+                })
+              })
+            })
+          }
+        }
+        ipfs.object.get(key, pathFn)
+      })
+    })
+  }
+}

src/core/index.js

@@ -17,6 +17,7 @@ const repo = require('./ipfs/repo')
 const init = require('./ipfs/init')
 const bootstrap = require('./ipfs/bootstrap')
 const config = require('./ipfs/config')
+const pinner = require('./ipfs/pinner')
 const block = require('./ipfs/block')
 const object = require('./ipfs/object')
 const libp2p = require('./ipfs/libp2p')
@@ -53,6 +54,7 @@ function IPFS (repoInstance) {
   this.init = init(this)
   this.bootstrap = bootstrap(this)
   this.config = config(this)
+  this.pinner = pinner(this)
   this.block = block(this)
   this.object = object(this)
   this.libp2p = libp2p(this)