diff --git a/src/cli/commands/pin/ls.js b/src/cli/commands/pin/ls.js new file mode 100644 index 0000000000..e640f605ab --- /dev/null +++ b/src/cli/commands/pin/ls.js @@ -0,0 +1,125 @@ +'use strict' + +const Command = require('ronin').Command +const utils = require('../../utils') +const debug = require('debug') +const log = debug('cli:pin') +log.error = debug('cli:pin:error') +const bs58 = require('bs58') + +const onError = (err) => { + if (err) { + console.error(err) + throw err + } +} + +module.exports = Command.extend({ + desc: 'Pins objects to local storage.', + + options: { + // The type of pinned keys to list. Can be + // "direct", "indirect", "recursive", or "all". + type: { + alias: 't', + type: 'string', + default: 'all' + }, + // Write just hashes of objects. + quiet: { + alias: 'q', + type: 'boolean', + default: false + } + }, + + run: (type, quiet, path) => { + utils.getIPFS((err, ipfs) => { + onError(err) + const types = ipfs.pinner.types + // load persistent pin set from datastore + ipfs.pinner.load(() => { + if (path) { + const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/) + if (!matched) { + onError(new Error('invalid ipfs ref path')) + } + const split = matched[1].split('/') + const rootHash = split[0] + const key = new Buffer(bs58.decode(rootHash)) + const links = split.slice(1, split.length) + const pathFn = (err, obj) => { + onError(err) + if (links.length) { + const linkName = links.shift() + const nextLink = obj.links.filter((link) => { + return (link.name === linkName) + }) + if (!nextLink.length) { + onError(new Error( + 'pin: no link named ' + linkName + + ' under ' + obj.toJSON().Hash + )) + } + const nextHash = nextLink[0].hash + ipfs.object.get(nextHash, pathFn) + } else { + ipfs.pinner.isPinnedWithType(obj.multihash(), type, (err, pinned, reason) => { + onError(err) + if (!pinned) { + onError(new Error('Path ' + path + ' is not pinned')) + } + if (reason !== types.direct && + reason !== types.recursive) { + reason = 'indirect through ' + reason + } + console.log(obj.toJSON().Hash + (quiet ? '' : ' ' + reason)) + }) + } + } + ipfs.object.get(key, pathFn) + } else { + const printDirect = () => { + ipfs.pinner.directKeyStrings().forEach((key) => { + console.log(key + (quiet ? '' : ' direct')) + }) + } + const printRecursive = () => { + ipfs.pinner.recursiveKeyStrings().forEach((key) => { + console.log(key + (quiet ? '' : ' recursive')) + }) + } + const printIndirect = () => { + ipfs.pinner.getIndirectKeys((err, keys) => { + onError(err) + keys.forEach((key) => { + console.log(key + (quiet ? '' : ' indirect')) + }) + }) + } + switch (type) { + case types.direct: + printDirect() + break + case types.recursive: + printRecursive() + break + case types.indirect: + printIndirect() + break + case types.all: + printDirect() + printRecursive() + printIndirect() + break + default: + onError(new Error( + "Invalid type '" + type + "', " + + 'must be one of {direct, indirect, recursive, all}' + )) + } + } + }) + }) + } +}) diff --git a/src/cli/commands/pin/rm.js b/src/cli/commands/pin/rm.js index 79aa33a7ba..547ecf3249 100644 --- a/src/cli/commands/pin/rm.js +++ b/src/cli/commands/pin/rm.js @@ -54,12 +54,18 @@ module.exports = Command.extend({ const nextHash = nextLink[0].hash ipfs.object.get(nextHash, pathFn) } else { - ipfs.pinner.unpin(obj.multihash(), recursive, (err) => { + ipfs.pinner.isPinned(obj.multihash(), (err, pinned, reason) => { onError(err) - // save modified pin state to datastore - ipfs.pinner.flush((err, root) => { + if (!pinned) { + onError(new Error('not pinned')) + } + ipfs.pinner.unpin(obj.multihash(), recursive, (err) => { onError(err) - console.log('unpinned ' + obj.toJSON().Hash) + // save modified pin state to datastore + ipfs.pinner.flush((err, root) => { + onError(err) + console.log('unpinned ' + obj.toJSON().Hash) + }) }) }) } diff --git a/src/core/ipfs/pinner.js b/src/core/ipfs/pinner.js index acc286646a..747c951432 100644 --- a/src/core/ipfs/pinner.js +++ b/src/core/ipfs/pinner.js @@ -5,35 +5,33 @@ const mDAG = require('ipfs-merkle-dag') const DAGNode = mDAG.DAGNode const pinnerUtils = require('./pinner-utils') +function keyString (key) { + return bs58.encode(key).toString() +} + function KeySet (keys) { // Buffers with identical data are still different objects, so // they need to be cast to strings to prevent duplicates in Sets - this.keys = new Set() - this.keyStrings = new Set() + this.keys = {} this.add = (key) => { - if (!this.has(key)) { - var keyString = bs58.encode(key).toString() - this.keyStrings.add(keyString) - this.keys.add(key) - } + this.keys[keyString(key)] = key } this.delete = (key) => { - var keyString = bs58.encode(key).toString() - this.keyStrings.delete(keyString) - this.keys.delete(key) + delete this.keys[keyString(key)] } this.clear = () => { - this.keys.clear() - this.keyStrings.clear() + this.keys = {} } this.has = (key) => { - return this.keyStrings.has(bs58.encode(key).toString()) + return (keyString(key) in this.keys) } this.toArray = () => { - return Array.from(this.keys) + return Object.keys(this.keys).map((hash) => { + return this.keys[hash] + }) } this.toStringArray = () => { - return Array.from(this.keyStrings) + return Object.keys(this.keys) } keys = keys || [] keys.forEach(this.add) @@ -93,8 +91,7 @@ module.exports = function (self) { return callback(err) } if (recursivePins.has(multihash)) { - return callback(bs58.encode(multihash).toString() + - ' already pinned recursively') + return callback(keyString(multihash) + ' already pinned recursively') } directPins.add(multihash) return callback(null) @@ -117,13 +114,12 @@ module.exports = function (self) { recursivePins.delete(multihash) return callback(null) } - return callback(bs58.encode(multihash).toString() + - ' is pinned recursively') + return callback(keyString(multihash) + ' is pinned recursively') case (pinner.types.direct): directPins.delete(multihash) return callback(null) default: - return callback(bs58.encode(multihash).toString() + + return callback(keyString(multihash) + ' is pinned indirectly under ' + reason) } }) @@ -189,7 +185,7 @@ module.exports = function (self) { if (has) { done() return callback( - null, true, bs58.encode(obj.multihash()).toString() + null, true, keyString(obj.multihash()) ) } else { checkedCount++ @@ -221,6 +217,34 @@ module.exports = function (self) { return recursivePins.toStringArray() }, + getIndirectKeys: (callback) => { + // callback (err, keys) + const indirectKeys = new KeySet() + const rKeys = pinner.recursiveKeys() + if (!rKeys.length) { return callback(null, []) } + var done = 0 + rKeys.forEach((multihash) => { + dagS.getRecursive(multihash, (err, objs) => { + if (callback.fired) { return } + if (err) { + callback.fired = true + return callback(err) + } + objs.forEach((obj) => { + const mh = obj.multihash() + if (!directPins.has(mh) && !recursivePins.has(mh)) { + // not already pinned recursively or directly + indirectKeys.add(mh) + } + }) + if (done === rKeys.length - 1) { + return callback(null, indirectKeys.toStringArray()) + } + done++ + }) + }) + }, + internalKeys: () => { return internalPins.toArray() }, @@ -270,6 +294,7 @@ module.exports = function (self) { }, load: (callback) => { + // callback (err) repo.datastore.get(pinDataStoreKey, (err, pseudoblock) => { if (err) { return callback(err) } var rootBytes = pseudoblock.data diff --git a/test/cli/test-pin.js b/test/cli/test-pin.js index 033015bb4b..1155e4dbdc 100644 --- a/test/cli/test-pin.js +++ b/test/cli/test-pin.js @@ -13,15 +13,14 @@ const _ = require('lodash') // `subLeaf const filenames = [ - 'root.json', 'leaf.json', 'branch.json', 'subLeaf.json' + 'root', 'leaf', 'branch', 'subLeaf' ] const keys = { - 'root.json': 'QmWQwS2Xh1SFGMPzUVYQ52b7RC7fTfiaPHm3ZyTRZuHmer', - 'leaf.json': 'QmaZoTQ6wFe7EtvaePBUeXavfeRqCAq3RUMomFxBpZLrLA', - 'branch.json': 'QmNxjjP7dtx6pzxWGBRCrgmjX3JqKL7uF2Kjx7ExiZDbSB', - 'subLeaf.json': 'QmUzzznkyQL7FjjBztG3D1tTjBuxeArLceDZnuSowUggXL' + root: 'QmWQwS2Xh1SFGMPzUVYQ52b7RC7fTfiaPHm3ZyTRZuHmer', + leaf: 'QmaZoTQ6wFe7EtvaePBUeXavfeRqCAq3RUMomFxBpZLrLA', + branch: 'QmNxjjP7dtx6pzxWGBRCrgmjX3JqKL7uF2Kjx7ExiZDbSB', + subLeaf: 'QmUzzznkyQL7FjjBztG3D1tTjBuxeArLceDZnuSowUggXL' } -const rootHash = keys['root.json'] describe('pin', function () { this.timeout(20 * 1000) @@ -35,7 +34,8 @@ describe('pin', function () { var doneCount = 0 filenames.forEach((filename) => { const hash = keys[filename] - nexpect.spawn('node', [bin, 'object', 'put', filesDir + filename], opts) + nexpect.spawn('node', [bin, 'object', 'put', + filesDir + filename + '.json'], opts) .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(stdout[0]).to.equal('added ' + hash) @@ -46,40 +46,106 @@ describe('pin', function () { describe('api offline', () => { it('add (recursively by default)', (done) => { - nexpect.spawn('node', [bin, 'pin', 'add', rootHash], opts) + nexpect.spawn('node', [bin, 'pin', 'add', keys.root], opts) .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout[0]).to.equal('pinned ' + rootHash + ' recursively') + expect(stdout[0]).to.equal('pinned ' + keys.root + ' recursively') done() }) }) it('add (direct)', (done) => { - nexpect.spawn('node', [bin, 'pin', 'add', '--recursive=false', - keys['leaf.json']], opts) + nexpect.spawn('node', [bin, 'pin', 'add', + '--recursive=false', keys.leaf], opts) .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout[0]).to.equal('pinned ' + keys['leaf.json'] + ' directly') + expect(stdout[0]).to.equal('pinned ' + keys.leaf + ' directly') done() }) }) + + it('ls (recursive)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', keys.root], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal(keys.root + ' recursive') + done() + }) + }) + it('ls (direct)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', keys.leaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal(keys.leaf + ' direct') + done() + }) + }) + it('ls (indirect)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', keys.subLeaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal( + keys.subLeaf + ' indirect through ' + keys.root + ) + done() + }) + }) + it('ls (all)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout.length).to.equal(4) + expect(stdout.indexOf(keys.root + ' recursive') >= 0).to.be.true + expect(stdout.indexOf(keys.leaf + ' direct') >= 0).to.be.true + expect(stdout.indexOf(keys.branch + ' indirect') >= 0).to.be.true + expect(stdout.indexOf(keys.subLeaf + ' indirect') >= 0).to.be.true + done() + }) + }) + it('ls (quiet)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', '--quiet=true'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout.length).to.equal(4) + filenames.forEach((filename) => { + expect(stdout.indexOf(keys[filename]) >= 0).to.be.true + }) + done() + }) + }) + it('rm (recursively by default)', (done) => { - nexpect.spawn('node', [bin, 'pin', 'rm', rootHash], opts) + nexpect.spawn('node', [bin, 'pin', 'rm', keys.root], opts) .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout[0]).to.equal('unpinned ' + rootHash) + expect(stdout[0]).to.equal('unpinned ' + keys.root) done() }) }) it('rm (direct)', (done) => { - nexpect.spawn('node', [bin, 'pin', 'rm', '--recursive=false', - keys['leaf.json']], opts) + nexpect.spawn('node', [bin, 'pin', 'rm', + '--recursive=false', keys.leaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal('unpinned ' + keys.leaf) + done() + }) + }) + + it('confirm removal', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls'], opts) .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout[0]).to.equal('unpinned ' + keys['leaf.json']) + expect(stdout.length).to.equal(0) done() }) })