From 6d46e2e9bf5d36dc7f8dcbc92b97347e193719d3 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 17 Dec 2018 10:18:55 +0000 Subject: [PATCH] feat: cid base option (#1552) Implements an option for the CLI, HTTP API and core (where appropriate) that will allow the user to pick the multibase encoding for any CIDs that are returned as strings. **NOTE** Using the CID base option in `bitswap`, `dag` and `object` APIs **WILL NOT** auto upgrade your CID to v1 if it is a v0 CID and **WILL NOT** apply the encoding you specified. This is because these APIs return IPLD objects with links and changing the version of the links would result in a different hash for the node if you were to re-add it. Also, the CID you used to retrieve the node wouldn't actually refer to the node you got back any longer. [Read this](https://github.com/ipfs/go-ipfs/issues/5349#issuecomment-445104823) for further context. This PR adds a `--cid-base` option to the following **CLI commands**: * `jsipfs bitswap stat` * `jsipfs bitswap unwant` * `jsipfs bitswap wantlist` * `jsipfs block put` * `jsipfs block stat` * `jsipfs add` * `jsipfs ls` * `jsipfs object get` * `jsipfs object links` * `jsipfs object new` * `jsipfs object patch add-link` * `jsipfs object patch append-data` * `jsipfs object patch rm-link` * `jsipfs object patch set-data` * `jsipfs object put` * `jsipfs object stat` * `jsipfs pin add` * `jsipfs pin ls` * `jsipfs pin rm` * `jsipfs resolve` Note: these two MFS commands _already_ implement the `--cid-base` option: * `jsipfs files ls` * `jsipfs files stat` It also adds `?cid-base=` query option to the following **HTTP endpoints**: * `/api/v0/bitswap/wantlist` * `/api/v0/bitswap/stat` * `/api/v0/bitswap/unwant` * `/api/v0/block/put` * `/api/v0/block/stat` * `/api/v0/add` * `/api/v0/ls` * `/api/v0/object/new` * `/api/v0/object/get` * `/api/v0/object/put` * `/api/v0/object/stat` * `/api/v0/object/links` * `/api/v0/object/patch/append-data` * `/api/v0/object/patch/set-data` * `/api/v0/object/patch/add-link` * `/api/v0/object/patch/rm-link` * `/api/v0/pin/ls` * `/api/v0/pin/add` * `/api/v0/pin/rm` * `/api/v0/resolve` It adds a `cidBase` option to the following **core functions**: * `resolve` License: MIT Signed-off-by: Alan Shaw --- package.json | 4 +- src/cli/commands/add.js | 17 +- src/cli/commands/bitswap/stat.js | 19 +- src/cli/commands/bitswap/unwant.js | 15 +- src/cli/commands/bitswap/wantlist.js | 17 +- src/cli/commands/block/get.js | 7 +- src/cli/commands/block/put.js | 32 +- src/cli/commands/block/rm.js | 12 +- src/cli/commands/block/stat.js | 19 +- src/cli/commands/ls.js | 36 +- src/cli/commands/object/get.js | 60 ++- src/cli/commands/object/links.js | 17 +- src/cli/commands/object/new.js | 18 +- src/cli/commands/object/patch/add-link.js | 22 +- src/cli/commands/object/patch/append-data.js | 23 +- src/cli/commands/object/patch/rm-link.js | 18 +- src/cli/commands/object/patch/set-data.js | 23 +- src/cli/commands/object/put.js | 19 +- src/cli/commands/object/stat.js | 6 +- src/cli/commands/pin/add.js | 17 +- src/cli/commands/pin/ls.js | 19 +- src/cli/commands/pin/rm.js | 16 +- src/cli/commands/resolve.js | 9 +- src/core/components/bitswap.js | 34 +- src/core/components/block.js | 21 +- .../files-regular/add-pull-stream.js | 20 +- .../files-regular/ls-pull-stream.js | 3 +- src/core/components/object.js | 2 + src/core/components/pin.js | 29 +- src/core/components/resolve.js | 5 +- src/http/api/resources/bitswap.js | 37 +- src/http/api/resources/block.js | 22 +- src/http/api/resources/files-regular.js | 22 +- src/http/api/resources/object.js | 174 +++++-- src/http/api/resources/pin.js | 35 +- src/http/api/resources/resolve.js | 9 +- src/http/api/routes/bitswap.js | 9 +- src/http/api/routes/block.js | 12 +- src/http/api/routes/object.js | 27 +- src/http/api/routes/pin.js | 9 +- src/utils/cid.js | 35 ++ test/cli/bitswap.js | 42 +- test/cli/block.js | 24 + test/cli/files.js | 9 + test/cli/ls.js | 15 + test/cli/object.js | 89 +++- test/cli/pin.js | 28 ++ test/cli/resolve.js | 24 +- test/http-api/inject/bitswap.js | 78 +++ test/http-api/inject/block.js | 85 +++- test/http-api/inject/files.js | 77 ++- test/http-api/inject/object.js | 469 ++++++++++++++++++ test/http-api/inject/pin.js | 169 +++++++ test/http-api/inject/resolve.js | 72 +++ 54 files changed, 1784 insertions(+), 347 deletions(-) create mode 100644 src/utils/cid.js create mode 100644 test/http-api/inject/resolve.js diff --git a/package.json b/package.json index d1f51beaab..9875580401 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "ipfs-bitswap": "~0.21.0", "ipfs-block": "~0.8.0", "ipfs-block-service": "~0.15.1", - "ipfs-http-client": "^28.0.1", + "ipfs-http-client": "^28.1.0", "ipfs-http-response": "~0.2.1", "ipfs-mfs": "~0.8.0", "ipfs-multipart": "~0.1.0", @@ -122,7 +122,7 @@ "ipld-git": "~0.2.2", "ipld-zcash": "~0.1.6", "ipns": "~0.4.3", - "is-ipfs": "~0.4.7", + "is-ipfs": "~0.4.8", "is-pull-stream": "~0.0.0", "is-stream": "^1.1.0", "joi": "^14.3.0", diff --git a/src/cli/commands/add.js b/src/cli/commands/add.js index 8b930434d6..1d29e46853 100644 --- a/src/cli/commands/add.js +++ b/src/cli/commands/add.js @@ -11,9 +11,9 @@ const getFolderSize = require('get-folder-size') const byteman = require('byteman') const waterfall = require('async/waterfall') const mh = require('multihashes') -const utils = require('../utils') -const print = require('../utils').print -const createProgressBar = require('../utils').createProgressBar +const multibase = require('multibase') +const { print, isDaemonOn, createProgressBar } = require('../utils') +const { cidToString } = require('../../utils/cid') function checkPath (inPath, recursive) { // This function is to check for the following possible inputs @@ -90,7 +90,7 @@ function addPipeline (index, addStream, list, argv) { sortBy(added, 'path') .reverse() .map((file) => { - const log = [ 'added', file.hash ] + const log = [ 'added', cidToString(file.hash, { base: argv.cidBase }) ] if (!quiet && file.path.length > 0) log.push(file.path) @@ -156,10 +156,15 @@ module.exports = { describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)', default: 0 }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + }, hash: { type: 'string', choices: Object.keys(mh.names), - describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)' + describe: 'Hash function to use. Will set CID version to 1 if used. (experimental)' }, quiet: { alias: 'q', @@ -202,7 +207,7 @@ module.exports = { chunker: argv.chunker } - if (options.enableShardingExperiment && utils.isDaemonOn()) { + if (options.enableShardingExperiment && isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } const ipfs = argv.ipfs diff --git a/src/cli/commands/bitswap/stat.js b/src/cli/commands/bitswap/stat.js index d7a26680f1..94060701eb 100644 --- a/src/cli/commands/bitswap/stat.js +++ b/src/cli/commands/bitswap/stat.js @@ -1,22 +1,29 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'stat', describe: 'Show some diagnostic information on the bitswap agent.', - builder: {}, + builder: { + 'cid-base': { + describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', + type: 'string', + choices: multibase.names + } + }, - handler (argv) { - argv.ipfs.bitswap.stat((err, stats) => { + handler ({ ipfs, cidBase }) { + ipfs.bitswap.stat((err, stats) => { if (err) { throw err } - stats.wantlist = stats.wantlist || [] - stats.wantlist = stats.wantlist.map(entry => entry['/']) + stats.wantlist = stats.wantlist.map(k => cidToString(k['/'], { base: cidBase, upgrade: false })) stats.peers = stats.peers || [] print(`bitswap status diff --git a/src/cli/commands/bitswap/unwant.js b/src/cli/commands/bitswap/unwant.js index ea676450fa..af0c77296b 100644 --- a/src/cli/commands/bitswap/unwant.js +++ b/src/cli/commands/bitswap/unwant.js @@ -1,6 +1,8 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'unwant ', @@ -12,14 +14,19 @@ module.exports = { alias: 'k', describe: 'Key to remove from your wantlist', type: 'string' + }, + 'cid-base': { + describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', + type: 'string', + choices: multibase.names } }, - handler (argv) { - argv.ipfs.bitswap.unwant(argv.key, (err) => { + handler ({ ipfs, key, cidBase }) { + ipfs.bitswap.unwant(key, (err) => { if (err) { throw err } - print(`Key ${argv.key} removed from wantlist`) + print(`Key ${cidToString(key, { base: cidBase, upgrade: false })} removed from wantlist`) }) } } diff --git a/src/cli/commands/bitswap/wantlist.js b/src/cli/commands/bitswap/wantlist.js index 9397f492ae..7c8e15d1a2 100644 --- a/src/cli/commands/bitswap/wantlist.js +++ b/src/cli/commands/bitswap/wantlist.js @@ -1,6 +1,8 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'wantlist [peer]', @@ -12,17 +14,20 @@ module.exports = { alias: 'p', describe: 'Specify which peer to show wantlist for.', type: 'string' + }, + 'cid-base': { + describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', + type: 'string', + choices: multibase.names } }, - handler (argv) { - argv.ipfs.bitswap.wantlist(argv.peer, (err, res) => { + handler ({ ipfs, peer, cidBase }) { + ipfs.bitswap.wantlist(peer, (err, list) => { if (err) { throw err } - res.Keys.forEach((cid) => { - print(cid['/']) - }) + list.Keys.forEach(k => print(cidToString(k['/'], { base: cidBase, upgrade: false }))) }) } } diff --git a/src/cli/commands/block/get.js b/src/cli/commands/block/get.js index 9fb61f4434..f5c32fe0e1 100644 --- a/src/cli/commands/block/get.js +++ b/src/cli/commands/block/get.js @@ -1,6 +1,5 @@ 'use strict' -const CID = require('cids') const print = require('../../utils').print module.exports = { @@ -10,10 +9,8 @@ module.exports = { builder: {}, - handler (argv) { - const cid = new CID(argv.key) - - argv.ipfs.block.get(cid, (err, block) => { + handler ({ ipfs, key }) { + ipfs.block.get(key, (err, block) => { if (err) { throw err } diff --git a/src/cli/commands/block/put.js b/src/cli/commands/block/put.js index 9d75a9c72f..a87204b46e 100644 --- a/src/cli/commands/block/put.js +++ b/src/cli/commands/block/put.js @@ -1,33 +1,19 @@ 'use strict' -const CID = require('cids') -const multihashing = require('multihashing-async') const bl = require('bl') const fs = require('fs') -const Block = require('ipfs-block') -const waterfall = require('async/waterfall') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') function addBlock (data, opts) { const ipfs = opts.ipfs - let cid - waterfall([ - (cb) => multihashing(data, opts.mhtype || 'sha2-256', cb), - (multihash, cb) => { - if (opts.format !== 'dag-pb' || opts.version !== 0) { - cid = new CID(1, opts.format || 'dag-pb', multihash) - } else { - cid = new CID(0, 'dag-pb', multihash) - } - - ipfs.block.put(new Block(data, cid), cb) - } - ], (err) => { + ipfs.block.put(data, opts, (err, block) => { if (err) { throw err } - print(cid.toBaseEncodedString()) + print(cidToString(block.cid, { base: opts.cidBase })) }) } @@ -52,8 +38,12 @@ module.exports = { }, version: { describe: 'cid version', - type: 'number', - default: 0 + type: 'number' + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, diff --git a/src/cli/commands/block/rm.js b/src/cli/commands/block/rm.js index 40e1951058..7b8a27db60 100644 --- a/src/cli/commands/block/rm.js +++ b/src/cli/commands/block/rm.js @@ -1,8 +1,6 @@ 'use strict' -const utils = require('../../utils') -const mh = require('multihashes') -const print = utils.print +const { print, isDaemonOn } = require('../../utils') module.exports = { command: 'rm ', @@ -11,18 +9,18 @@ module.exports = { builder: {}, - handler (argv) { - if (utils.isDaemonOn()) { + handler ({ ipfs, key }) { + if (isDaemonOn()) { // TODO implement this once `js-ipfs-http-client` supports it throw new Error('rm block with daemon running is not yet implemented') } - argv.ipfs.block.rm(mh.fromB58String(argv.key), (err) => { + ipfs.block.rm(key, (err) => { if (err) { throw err } - print('removed ' + argv.key) + print('removed ' + key) }) } } diff --git a/src/cli/commands/block/stat.js b/src/cli/commands/block/stat.js index 07190fd225..9c9bdb5e21 100644 --- a/src/cli/commands/block/stat.js +++ b/src/cli/commands/block/stat.js @@ -1,22 +1,29 @@ 'use strict' -const CID = require('cids') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'stat ', describe: 'Print information of a raw IPFS block', - builder: {}, + builder: { + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + } + }, - handler (argv) { - argv.ipfs.block.stat(new CID(argv.key), (err, stats) => { + handler ({ ipfs, key, cidBase }) { + ipfs.block.stat(key, (err, stats) => { if (err) { throw err } - print('Key: ' + stats.key) + print('Key: ' + cidToString(stats.key, { base: cidBase })) print('Size: ' + stats.size) }) } diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js index 97ba7ca692..66667326bd 100644 --- a/src/cli/commands/ls.js +++ b/src/cli/commands/ls.js @@ -1,6 +1,8 @@ 'use strict' -const utils = require('../utils') +const multibase = require('multibase') +const { print, rightpad } = require('../utils') +const { cidToString } = require('../../utils/cid') module.exports = { command: 'ls ', @@ -24,33 +26,41 @@ module.exports = { desc: 'Resolve linked objects to find out their types. (not implemented yet)', type: 'boolean', default: false // should be true when implemented + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, - handler (argv) { - let path = argv.key - if (path.startsWith('/ipfs/')) { - path = path.replace('/ipfs/', '') - } - - argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => { + handler ({ ipfs, key, recursive, headers, cidBase }) { + ipfs.ls(key, { recursive }, (err, links) => { if (err) { throw err } - if (argv.headers) { + links = links.map(file => Object.assign(file, { hash: cidToString(file.hash, { base: cidBase }) })) + + if (headers) { links = [{ hash: 'Hash', size: 'Size', name: 'Name' }].concat(links) } const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) + let pathParts = key.split('/') + + if (key.startsWith('/ipfs/')) { + pathParts = pathParts.slice(2) + } + links.forEach(link => { const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name - const padding = link.depth - path.split('/').length - utils.print( - utils.rightpad(link.hash, multihashWidth + 1) + - utils.rightpad(link.size || '', sizeWidth + 1) + + const padding = link.depth - pathParts.length + print( + rightpad(link.hash, multihashWidth + 1) + + rightpad(link.size || '', sizeWidth + 1) + ' '.repeat(padding) + fileName ) }) diff --git a/src/cli/commands/object/get.js b/src/cli/commands/object/get.js index 548ae4f508..456449492c 100644 --- a/src/cli/commands/object/get.js +++ b/src/cli/commands/object/get.js @@ -1,7 +1,8 @@ 'use strict' -const print = require('../../utils').print -const dagPB = require('ipld-dag-pb') +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'get ', @@ -14,43 +15,38 @@ module.exports = { default: 'base64' }, 'cid-base': { - default: 'base58btc', - describe: 'CID base to use.' + describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', + type: 'string', + choices: multibase.names } }, - handler (argv) { - argv.ipfs.object.get(argv.key, { enc: 'base58' }, (err, node) => { + handler ({ ipfs, key, dataEncoding, cidBase }) { + ipfs.object.get(key, { enc: 'base58' }, (err, node) => { if (err) { throw err } - dagPB.util.cid(node, (err, result) => { - if (err) { - throw err - } - - let data = node.data - - if (Buffer.isBuffer(data)) { - data = node.data.toString(argv.dataEncoding || undefined) - } - - const answer = { - Data: data, - Hash: result.toBaseEncodedString(argv.cidBase), - Size: node.size, - Links: node.links.map((l) => { - return { - Name: l.name, - Size: l.size, - Hash: l.cid.toBaseEncodedString(argv.cidBase) - } - }) - } - - print(JSON.stringify(answer)) - }) + let data = node.data + + if (Buffer.isBuffer(data)) { + data = node.data.toString(dataEncoding || undefined) + } + + const answer = { + Data: data, + Hash: cidToString(key, { base: cidBase, upgrade: false }), + Size: node.size, + Links: node.links.map((l) => { + return { + Name: l.name, + Size: l.size, + Hash: cidToString(l.cid, { base: cidBase, upgrade: false }) + } + }) + } + + print(JSON.stringify(answer)) }) } } diff --git a/src/cli/commands/object/links.js b/src/cli/commands/object/links.js index 9a3a3e203e..768e42150c 100644 --- a/src/cli/commands/object/links.js +++ b/src/cli/commands/object/links.js @@ -1,6 +1,8 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'links ', @@ -9,21 +11,20 @@ module.exports = { builder: { 'cid-base': { - default: 'base58btc', - describe: 'CID base to use.' + describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', + type: 'string', + choices: multibase.names } }, - handler (argv) { - argv.ipfs.object.links(argv.key, { - enc: 'base58' - }, (err, links) => { + handler ({ ipfs, key, cidBase }) { + ipfs.object.links(key, { enc: 'base58' }, (err, links) => { if (err) { throw err } links.forEach((link) => { - print(`${link.cid.toBaseEncodedString(argv.cidBase)} ${link.size} ${link.name}`) + print(`${cidToString(link.cid, { base: cidBase, upgrade: false })} ${link.size} ${link.name}`) }) }) } diff --git a/src/cli/commands/object/new.js b/src/cli/commands/object/new.js index fc8543fd25..d48f4c69ee 100644 --- a/src/cli/commands/object/new.js +++ b/src/cli/commands/object/new.js @@ -1,9 +1,8 @@ 'use strict' -const debug = require('debug') -const log = debug('cli:object') -log.error = debug('cli:object:error') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'new [